SourceIndex 0.1.0__py3-none-any.whl

sourceindex/__init__.py

@@ -0,0 +1,30 @@
+__version__ = "0.1.0"
+
+# Surfaced at the top level so the CLI's argparse setup can use it for
+# --workers help/default without importing the build pipeline (which
+# transitively pulls litellm and inflates `sourceindex statusline` startup
+# from ~20ms to ~2.3s on every render tick).
+REALTIME_MAX_WORKERS = 128
+
+
+# Suppress LiteLLM's bedrock/sagemaker pre-load warnings. They fire at import
+# from litellm.litellm_core_utils.common_utils when botocore isn't installed
+# — a baseline "expected" state for our shipped wheel (we route through
+# OpenAI/Anthropic/OpenRouter, never Bedrock or SageMaker, and don't pull in
+# a 100MB+ botocore just to satisfy that startup probe). Installed at package
+# import so it runs before any submodule imports litellm.
+import logging as _logging
+
+
+class _LiteLLMBotocorePreloadFilter(_logging.Filter):
+    _SUPPRESS = (
+        "could not pre-load bedrock-runtime",
+        "could not pre-load sagemaker-runtime",
+    )
+
+    def filter(self, record: _logging.LogRecord) -> bool:
+        msg = record.getMessage()
+        return not any(s in msg for s in self._SUPPRESS)
+
+
+_logging.getLogger("LiteLLM").addFilter(_LiteLLMBotocorePreloadFilter())

sourceindex/build/__init__.py

@@ -0,0 +1,592 @@
+"""Tier-1 + tier-2 index build/update pipeline.
+
+The ``_with_store`` variants are what the daemon calls. The legacy
+``index_dir``-based entry points wrap a ``PlaintextStore`` for tests
+and the eval-dumped cache flow."""
+
+import os
+import subprocess
+import time
+from pathlib import Path
+
+from ..lib.languages import (
+    Language,
+    all_extensions,
+    detect_languages,
+    get_languages,
+    language_for_path,
+)
+from ..lib.env import env_truthy
+from ..lib.llm import LLMUsage
+from ..lib.log import get_logger, log_file_error
+from .indexer import (
+    BATCH_MIN_FILES,
+    DEFAULT_MODEL,
+    REALTIME_MAX_WORKERS,
+    emit_status,
+    index_by_hash,
+    index_file,
+)
+from .state import (
+    BUILD_META_FILENAME,
+    load_file_index,
+    load_summary_cache,
+    merge_build_meta,
+    save_file_index,
+    save_summary_cache,
+    try_summary_cache_hit,
+    utcnow_iso,
+    write_build_meta,
+    write_details,
+)
+from .walker import (
+    collect_source_files,
+    detect_changes_by_mtime,
+    hash_file_content,
+)
+
+_log = get_logger(__name__)
+
+
+def _resolve_plaintext_store(repo_root: Path, index_dir: Path | None):
+    """Always returns a PlaintextStore — the legacy-API back-compat shim
+    for callers that haven't migrated to the daemon path."""
+    from ..daemon import IndexDir, PlaintextStore
+    path = index_dir if index_dir is not None else IndexDir.for_repo(repo_root).path
+    return PlaintextStore(path)
+
+
+def build_index(
+    repo_root: Path,
+    index_dir: Path | None = None,
+    model: str = DEFAULT_MODEL,
+    api_key: str | None = None,
+    languages: list[Language] | None = None,
+    *,
+    workers: int = REALTIME_MAX_WORKERS,
+) -> None:
+    """Legacy entry point: build full tier-1 and tier-2 index for a repository.
+
+    Args:
+        repo_root: Path to the repository root.
+        index_dir: Where to write the index (default: repo_root/.sourceindex).
+        model: LLM model to use for summarization.
+        api_key: Anthropic API key for batch mode. If None, uses `claude -p`.
+        languages: Language objects to index. If None, auto-detects.
+        workers: Max concurrent LLM requests in batch mode.
+    """
+    store = _resolve_plaintext_store(repo_root, index_dir)
+    build_index_with_store(
+        repo_root,
+        store,
+        model=model,
+        api_key=api_key,
+        languages=languages,
+        workers=workers,
+    )
+
+
+def build_index_with_store(
+    repo_root: Path,
+    store,
+    *,
+    model: str | None = None,
+    api_key: str | None = None,
+    languages: list[Language] | list[str] | None = None,
+    workers: int | None = None,
+) -> dict:
+    model = model or DEFAULT_MODEL
+    workers = workers or REALTIME_MAX_WORKERS
+    languages_resolved = _resolve_languages_arg(repo_root, languages)
+
+    store.ensure_initialized()
+
+    started_at = utcnow_iso()
+    start_time = time.time()
+    total_usage = LLMUsage()
+
+    _log.debug(
+        "Build starting",
+        extra={
+            "event": "build.start",
+            "repo_root": str(repo_root),
+            "model": model,
+            "workers": workers,
+            "language_count": len(languages_resolved),
+        },
+    )
+
+    source_files = collect_source_files(repo_root, languages_resolved)
+    if not source_files:
+        _log.info("No source files found.", extra={"event": "build.files_found", "count": 0})
+        store.write_text("index.md", "(no source files found)\n")
+        write_build_meta(
+            store,
+            model=model,
+            files_indexed=0,
+            files_from_cache=0,
+            usage=total_usage,
+            elapsed_s=time.time() - start_time,
+            started_at=started_at,
+        )
+        return {"files_indexed": 0, "files_from_cache": 0, "cost_usd": 0.0,
+                "elapsed_s": time.time() - start_time}
+
+    _log.info(
+        f"Found {len(source_files)} source files to index.",
+        extra={"event": "build.files_found", "count": len(source_files)},
+    )
+
+    summary_cache = load_summary_cache(store)
+
+    file_contents: dict[str, str] = {}
+    file_hashes: dict[str, str] = {}
+    file_langs: dict[str, Language] = {}
+    needs_llm: dict[str, str] = {}
+    oneliners: dict[str, str] = {}
+
+    for rel_path in source_files:
+        full_path = repo_root / rel_path
+        try:
+            content = full_path.read_text(errors="replace")
+        except OSError as e:
+            log_file_error(_log, rel_path, e, phase="build.read")
+            continue
+        if not content.strip():
+            continue
+
+        lang = language_for_path(rel_path, languages_resolved)
+        if lang is None:
+            continue
+        h = hash_file_content(content)
+        file_hashes[rel_path] = h
+        file_contents[rel_path] = content
+        file_langs[rel_path] = lang
+
+        hit = try_summary_cache_hit(summary_cache, h, lang, store, rel_path)
+        if hit is not None:
+            oneliners[rel_path] = hit[0]
+        else:
+            needs_llm[rel_path] = content
+
+    cache_hits = len(oneliners)
+    cache_misses = len(needs_llm)
+    _log.info(
+        f"Summary cache: {cache_hits} hits, {cache_misses} misses.",
+        extra={
+            "event": "build.cache_summary",
+            "cache_hits": cache_hits,
+            "cache_misses": cache_misses,
+        },
+    )
+
+    if api_key and cache_misses > 0:
+        hash_to_content: dict[str, tuple[str, str, Language]] = {}
+        for rel_path, content in needs_llm.items():
+            hash_to_content[file_hashes[rel_path]] = (content, rel_path, file_langs[rel_path])
+
+        hash_to_parsed, batch_usage = index_by_hash(
+            hash_to_content,
+            summary_cache=summary_cache,
+            model=model,
+            api_key=api_key,
+            workers=workers,
+            label="index",
+        )
+        total_usage.add(batch_usage)
+
+        emit_status("[finalize] writing tier-2 details and saving cache...")
+        for rel_path in needs_llm:
+            h = file_hashes[rel_path]
+            if h in hash_to_parsed:
+                oneliner, functions = hash_to_parsed[h]
+                if oneliner:
+                    oneliners[rel_path] = oneliner
+                if file_langs[rel_path].build_tier2:
+                    write_details(store, rel_path, functions)
+            else:
+                oneliners[rel_path] = "(LLM summary failed)"
+                if file_langs[rel_path].build_tier2:
+                    write_details(store, rel_path, "")
+    else:
+        for rel_path, content in needs_llm.items():
+            _log.info(f"[{rel_path}] Indexing...", extra={"event": "build.file_indexing", "path": rel_path})
+            lang = file_langs[rel_path]
+            oneliner, functions, usage = index_file(content, rel_path, lang, model=model, api_key=api_key)
+            total_usage.add(usage)
+            oneliners[rel_path] = oneliner
+            summary_cache[file_hashes[rel_path]] = {
+                "oneliner": oneliner,
+                "functions": functions,
+            }
+            if lang.build_tier2:
+                write_details(store, rel_path, functions)
+
+    emit_status("[finalize] writing index.md and per-file metadata...")
+    index_lines = []
+    for rel_path in source_files:
+        if rel_path in oneliners:
+            index_lines.append(f"{rel_path} — {oneliners[rel_path]}")
+
+    store.write_text("index.md", "\n".join(index_lines) + "\n")
+
+    if cache_misses > 0:
+        save_summary_cache(store, summary_cache)
+
+    file_index_map: dict[str, dict] = {}
+    for rel_path in source_files:
+        if rel_path not in oneliners:
+            continue
+        try:
+            mtime = (repo_root / rel_path).stat().st_mtime_ns
+        except OSError as e:
+            log_file_error(_log, rel_path, e, phase="build.stat")
+            continue
+        file_index_map[rel_path] = {
+            "mtime_ns": mtime,
+            "hash": file_hashes.get(rel_path, ""),
+        }
+    save_file_index(store, file_index_map)
+
+    elapsed_s = time.time() - start_time
+    write_build_meta(
+        store,
+        model=model,
+        files_indexed=cache_misses,
+        files_from_cache=cache_hits,
+        usage=total_usage,
+        elapsed_s=elapsed_s,
+        started_at=started_at,
+    )
+
+    _log.info(
+        f"Done. Index: {store.path_for('index.md')} ({len(index_lines)} files, "
+        f"{cache_hits} from cache)",
+        extra={
+            "event": "build.done",
+            "index_path": str(store.path_for("index.md")),
+            "files_in_index": len(index_lines),
+            "files_indexed": cache_misses,
+            "files_from_cache": cache_hits,
+            "elapsed_s": round(elapsed_s, 3),
+            "input_tokens": total_usage.input_tokens,
+            "output_tokens": total_usage.output_tokens,
+            "cost_usd": round(total_usage.cost_usd, 6),
+        },
+    )
+    return {
+        "files_indexed": cache_misses,
+        "files_from_cache": cache_hits,
+        "cost_usd": round(total_usage.cost_usd, 6),
+        "elapsed_s": round(elapsed_s, 3),
+    }
+
+
+def update_index(
+    repo_root: Path,
+    index_dir: Path | None = None,
+    changed_files: list[str] | None = None,
+    model: str = DEFAULT_MODEL,
+    languages: list[Language] | None = None,
+    api_key: str | None = None,
+    *,
+    persist_file_index: bool = True,
+    workers: int = REALTIME_MAX_WORKERS,
+) -> None:
+    store = _resolve_plaintext_store(repo_root, index_dir)
+    update_index_with_store(
+        repo_root,
+        store,
+        changed_files=changed_files,
+        model=model,
+        languages=languages,
+        api_key=api_key,
+        persist_file_index=persist_file_index,
+        workers=workers,
+    )
+
+
+def update_index_with_store(
+    repo_root: Path,
+    store,
+    *,
+    changed_files: list[str] | None = None,
+    model: str | None = None,
+    languages: list[Language] | list[str] | None = None,
+    api_key: str | None = None,
+    persist_file_index: bool = True,
+    workers: int | None = None,
+) -> dict:
+    model = model or DEFAULT_MODEL
+    workers = workers or REALTIME_MAX_WORKERS
+    languages_resolved = _resolve_languages_arg(repo_root, languages)
+
+    if not store.exists("index.md"):
+        _log.info(
+            "No existing index found. Run `sourceindex init` first.",
+            extra={"event": "update.no_index", "index_path": str(store.path_for("index.md"))},
+        )
+        return {"files_indexed": 0, "elapsed_s": 0.0, "cost_usd": 0.0}
+
+    exts = all_extensions(languages_resolved)
+
+    if changed_files is None:
+        try:
+            pathspecs = [f"*{ext}" for ext in exts]
+            result = subprocess.run(
+                ["git", "diff", "--name-only", "HEAD~1", "HEAD", "--", *pathspecs],
+                capture_output=True, text=True, cwd=repo_root,
+            )
+            changed_files = [f.strip() for f in result.stdout.strip().split("\n") if f.strip()]
+        except Exception as e:
+            _log.warning(
+                "Could not detect changed files via git.",
+                extra={
+                    "event": "update.git_diff_failed",
+                    "error_type": type(e).__name__,
+                    "error": str(e),
+                },
+            )
+            return {"files_indexed": 0, "elapsed_s": 0.0, "cost_usd": 0.0}
+
+    changed_files = [f for f in changed_files if f.lower().endswith(exts)]
+
+    if not changed_files:
+        _log.info(
+            "No changed source files to update.",
+            extra={"event": "update.no_changes"},
+        )
+        return {"files_indexed": 0, "elapsed_s": 0.0, "cost_usd": 0.0}
+
+    _log.info(
+        f"Updating {len(changed_files)} changed files...",
+        extra={"event": "update.start", "changed_count": len(changed_files)},
+    )
+
+    started_at = utcnow_iso()
+    start_time = time.time()
+    total_usage = LLMUsage()
+    files_indexed = 0
+    summary_cache = load_summary_cache(store)
+    file_index_map = load_file_index(store) if persist_file_index else {}
+
+    existing_index: dict[str, str] = {}
+    for line in store.read_text("index.md").strip().split("\n"):
+        if " — " in line:
+            path, oneliner = line.split(" — ", 1)
+            existing_index[path] = oneliner
+
+    needs_llm: dict[str, tuple[str, Language, int, str]] = {}
+
+    for rel_path in changed_files:
+        full_path = repo_root / rel_path
+        if not full_path.is_file():
+            existing_index.pop(rel_path, None)
+            file_index_map.pop(rel_path, None)
+            store.unlink(f"details/{rel_path}.md", missing_ok=True)
+            _log.info(
+                f"  Removed: {rel_path}",
+                extra={"event": "update.file_removed", "path": rel_path},
+            )
+            continue
+
+        try:
+            st = full_path.stat()
+            content = full_path.read_text(errors="replace")
+        except OSError as e:
+            log_file_error(_log, rel_path, e, phase="update.read")
+            continue
+
+        if not content.strip():
+            continue
+
+        lang = language_for_path(rel_path, languages_resolved)
+        if lang is None:
+            continue
+
+        h = hash_file_content(content)
+        hit = try_summary_cache_hit(summary_cache, h, lang, store, rel_path)
+        if hit is not None:
+            existing_index[rel_path] = hit[0]
+            file_index_map[rel_path] = {"mtime_ns": st.st_mtime_ns, "hash": h}
+            _log.info(
+                f"  Cached:   {rel_path}",
+                extra={"event": "update.cache_hit", "path": rel_path},
+            )
+            continue
+
+        needs_llm[rel_path] = (content, lang, st.st_mtime_ns, h)
+
+    if api_key and len(needs_llm) >= BATCH_MIN_FILES:
+        hash_to_content: dict[str, tuple[str, str, Language]] = {}
+        for rel_path, (content, lang, _, h) in needs_llm.items():
+            hash_to_content[h] = (content, rel_path, lang)
+        _log.info(
+            f"  Concurrent indexing: {len(needs_llm)} files "
+            f"({len(hash_to_content)} unique)...",
+            extra={
+                "event": "update.concurrent_indexing",
+                "files": len(needs_llm),
+                "unique": len(hash_to_content),
+            },
+        )
+        hash_to_parsed, batch_usage = index_by_hash(
+            hash_to_content,
+            summary_cache=summary_cache,
+            model=model,
+            api_key=api_key,
+            workers=workers,
+            label="update",
+        )
+        total_usage.add(batch_usage)
+
+        for rel_path, (_, lang, mtime_ns, h) in needs_llm.items():
+            if h in hash_to_parsed:
+                oneliner, functions = hash_to_parsed[h]
+            else:
+                oneliner, functions = "(LLM summary failed)", ""
+            if lang.build_tier2:
+                write_details(store, rel_path, functions)
+            existing_index[rel_path] = oneliner
+            file_index_map[rel_path] = {"mtime_ns": mtime_ns, "hash": h}
+            files_indexed += 1
+    else:
+        for rel_path, (content, lang, mtime_ns, h) in needs_llm.items():
+            _log.info(
+                f"  Updating: {rel_path}",
+                extra={"event": "update.file_indexing", "path": rel_path},
+            )
+            oneliner, functions, usage = index_file(content, rel_path, lang, model=model, api_key=api_key)
+            total_usage.add(usage)
+            files_indexed += 1
+            summary_cache[h] = {"oneliner": oneliner, "functions": functions}
+            if lang.build_tier2:
+                write_details(store, rel_path, functions)
+            existing_index[rel_path] = oneliner
+            file_index_map[rel_path] = {"mtime_ns": mtime_ns, "hash": h}
+
+    index_lines = [f"{p} — {o}" for p, o in sorted(existing_index.items())]
+    store.write_text("index.md", "\n".join(index_lines) + "\n")
+    save_summary_cache(store, summary_cache)
+    if persist_file_index:
+        save_file_index(store, file_index_map)
+
+    elapsed_s = time.time() - start_time
+    merge_build_meta(
+        store,
+        model=model,
+        files_indexed=files_indexed,
+        usage=total_usage,
+        elapsed_s=elapsed_s,
+        started_at=started_at,
+    )
+
+    _log.info(
+        f"Updated {len(changed_files)} files.",
+        extra={
+            "event": "update.done",
+            "changed_count": len(changed_files),
+            "files_indexed": files_indexed,
+            "elapsed_s": round(elapsed_s, 3),
+            "input_tokens": total_usage.input_tokens,
+            "output_tokens": total_usage.output_tokens,
+            "cost_usd": round(total_usage.cost_usd, 6),
+        },
+    )
+    return {
+        "files_indexed": files_indexed,
+        "elapsed_s": round(elapsed_s, 3),
+        "cost_usd": round(total_usage.cost_usd, 6),
+    }
+
+
+def refresh_index_lazy(
+    repo_root: Path,
+    index_dir: Path | None = None,
+    *,
+    store=None,
+    model: str = DEFAULT_MODEL,
+    languages: list[Language] | None = None,
+    api_key: str | None = None,
+    workers: int = REALTIME_MAX_WORKERS,
+) -> bool:
+    """Pick up working-tree edits between commits. Pass ``store`` (daemon
+    path) or ``index_dir`` (legacy); store wins when both are supplied."""
+    if env_truthy("SOURCEINDEX_NO_AUTO_REFRESH"):
+        _log.debug(
+            "Lazy refresh disabled by SOURCEINDEX_NO_AUTO_REFRESH",
+            extra={"event": "lazy_refresh.disabled"},
+        )
+        return False
+
+    if store is None:
+        store = _resolve_plaintext_store(repo_root, index_dir)
+    if not store.exists("index.md"):
+        return False  # cold builds belong to `init`, not the search path
+
+    if languages is None:
+        languages = detect_languages(repo_root) or get_languages()
+
+    changed, updated, deleted, stored = detect_changes_by_mtime(
+        repo_root, store, languages
+    )
+
+    real_work = changed + deleted
+    if not real_work:
+        if updated != stored:
+            save_file_index(store, updated)
+            _log.debug(
+                "Lazy refresh: touch-only mtime bumps persisted",
+                extra={"event": "lazy_refresh.touch_only", "count": len(updated)},
+            )
+            return True
+        return False
+
+    from ..lib.llm import resolve_api_key
+    resolved_key = resolve_api_key(api_key)
+    if not resolved_key:
+        _log.warning(
+            f"Lazy refresh skipped: SOURCEINDEX_API_KEY not set; "
+            f"{len(changed)} changed file(s) detected.",
+            extra={
+                "event": "lazy_refresh.skipped",
+                "reason": "no_api_key",
+                "changed_count": len(changed),
+            },
+        )
+        return False
+
+    _log.debug(
+        "Lazy refresh starting",
+        extra={
+            "event": "lazy_refresh.applied",
+            "changed_count": len(changed),
+            "deleted_count": len(deleted),
+        },
+    )
+    update_index_with_store(
+        repo_root,
+        store,
+        changed_files=real_work,
+        model=model,
+        languages=languages,
+        api_key=resolved_key,
+        persist_file_index=False,
+        workers=workers,
+    )
+    save_file_index(store, updated)
+    return True
+
+
+def _resolve_languages_arg(
+    repo_root: Path,
+    languages: list[Language] | list[str] | None,
+) -> list[Language]:
+    # RPC callers pass list[str]; in-process callers pass list[Language].
+    if languages is None:
+        return detect_languages(repo_root) or get_languages()
+    if not languages:
+        return get_languages()
+    if isinstance(languages[0], str):
+        return get_languages(languages)  # type: ignore[arg-type]
+    return languages  # type: ignore[return-value]