agent-coderag 1.3.3__tar.gz → 1.3.4__tar.gz

{agent_coderag-1.3.3/agent_coderag.egg-info → agent_coderag-1.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agent-coderag
-Version: 1.3.3
+Version: 1.3.4
 Summary: Lightweight semantic code search and distillation utility for AI coding agents. It solves the API knowledge gap via real-time local signature extraction and intent analysis without PyTorch. Optimized for token efficiency, it compresses codebase context into compact semantic summaries stored in a local DuckDB vector similarity index.
 Author-email: Igor Boloban <naranor@gmail.com>
 License: MIT
@@ -119,6 +119,18 @@ If you don't configure an LLM provider, agent-coderag works in **100% Offline Mo
 - **Search:** Remains fast and accurate.
 - **Distillation:** Instead of AI-generated summaries, the system uses code signatures and entity names as fallback metadata. No data ever leaves your machine.
 
+Remote embeddings are **not** 100% Offline Mode. If you set `embedding_base` + `embedding_model`, `sync` / `search` / `rebuild` need the network. Embedder choice is process-global (`config.json`); after a remote model or dimension change, run `agent-coderag rebuild` (or delete that `--db`) for **each** project index.
+
+```bash
+agent-coderag config \
+  --embedding-url "http://localhost:8081/v1" \
+  --embedding-model "text-embedding-3-small" \
+  --embedding-key "your-api-key" \
+  --embedding-provider "openai"
+
+agent-coderag config --clear-embedding
+```
+
 ### First Sync & Search
 ```bash
 # Index your entire project (respects .gitignore automatically)

{agent_coderag-1.3.3 → agent_coderag-1.3.4}/README.md

@@ -73,6 +73,18 @@ If you don't configure an LLM provider, agent-coderag works in **100% Offline Mo
 - **Search:** Remains fast and accurate.
 - **Distillation:** Instead of AI-generated summaries, the system uses code signatures and entity names as fallback metadata. No data ever leaves your machine.
 
+Remote embeddings are **not** 100% Offline Mode. If you set `embedding_base` + `embedding_model`, `sync` / `search` / `rebuild` need the network. Embedder choice is process-global (`config.json`); after a remote model or dimension change, run `agent-coderag rebuild` (or delete that `--db`) for **each** project index.
+
+```bash
+agent-coderag config \
+  --embedding-url "http://localhost:8081/v1" \
+  --embedding-model "text-embedding-3-small" \
+  --embedding-key "your-api-key" \
+  --embedding-provider "openai"
+
+agent-coderag config --clear-embedding
+```
+
 ### First Sync & Search
 ```bash
 # Index your entire project (respects .gitignore automatically)

{agent_coderag-1.3.3 → agent_coderag-1.3.4/agent_coderag.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agent-coderag
-Version: 1.3.3
+Version: 1.3.4
 Summary: Lightweight semantic code search and distillation utility for AI coding agents. It solves the API knowledge gap via real-time local signature extraction and intent analysis without PyTorch. Optimized for token efficiency, it compresses codebase context into compact semantic summaries stored in a local DuckDB vector similarity index.
 Author-email: Igor Boloban <naranor@gmail.com>
 License: MIT
@@ -119,6 +119,18 @@ If you don't configure an LLM provider, agent-coderag works in **100% Offline Mo
 - **Search:** Remains fast and accurate.
 - **Distillation:** Instead of AI-generated summaries, the system uses code signatures and entity names as fallback metadata. No data ever leaves your machine.
 
+Remote embeddings are **not** 100% Offline Mode. If you set `embedding_base` + `embedding_model`, `sync` / `search` / `rebuild` need the network. Embedder choice is process-global (`config.json`); after a remote model or dimension change, run `agent-coderag rebuild` (or delete that `--db`) for **each** project index.
+
+```bash
+agent-coderag config \
+  --embedding-url "http://localhost:8081/v1" \
+  --embedding-model "text-embedding-3-small" \
+  --embedding-key "your-api-key" \
+  --embedding-provider "openai"
+
+agent-coderag config --clear-embedding
+```
+
 ### First Sync & Search
 ```bash
 # Index your entire project (respects .gitignore automatically)

{agent_coderag-1.3.3 → agent_coderag-1.3.4}/agent_coderag.egg-info/SOURCES.txt

@@ -36,6 +36,8 @@ code_rag/entry/cli.py
 code_rag/intelligence/__init__.py
 code_rag/intelligence/distiller.py
 code_rag/intelligence/embedder.py
+code_rag/intelligence/factory.py
+code_rag/intelligence/openai_embedder.py
 code_rag/parsers/__init__.py
 code_rag/parsers/languages.py
 code_rag/parsers/multi_parser.py
@@ -51,8 +53,10 @@ code_rag/storage/__init__.py
 code_rag/storage/duckdb_impl.py
 tests/test_api_client.py
 tests/test_api_models.py
+tests/test_api_rebuild_wipe.py
 tests/test_cli.py
 tests/test_cli_detailed.py
+tests/test_cli_embeddings.py
 tests/test_cli_json_parity.py
 tests/test_code_rag_simple.py
 tests/test_csharp_discovery_detailed.py
@@ -61,18 +65,29 @@ tests/test_discovery_manager_detailed.py
 tests/test_discovery_providers_extra.py
 tests/test_discovery_python.py
 tests/test_distiller.py
+tests/test_distiller_config_embedding.py
 tests/test_distiller_extra.py
 tests/test_embedder.py
 tests/test_embedder_detailed.py
+tests/test_embedder_factory.py
+tests/test_embedder_interface.py
+tests/test_embedder_stubs.py
+tests/test_factory_async.py
 tests/test_interfaces.py
 tests/test_js_discovery_detailed.py
 tests/test_languages.py
+tests/test_local_onnx_embedder.py
 tests/test_manager_detailed.py
+tests/test_manager_embeddings.py
 tests/test_manager_extra.py
 tests/test_models.py
 tests/test_multi_parser.py
+tests/test_openai_embedder.py
 tests/test_readme_library_usage.py
+tests/test_readme_offline_embeddings.py
 tests/test_rust_discovery_detailed.py
 tests/test_storage_detailed.py
+tests/test_storage_embeddings.py
+tests/test_sync_worker_errors.py
 tests/test_tree_sitter_parser.py
 tests/test_utils_detailed.py

{agent_coderag-1.3.3 → agent_coderag-1.3.4}/code_rag/api/client.py

@@ -2,6 +2,7 @@ from pathlib import Path
 from typing import Optional, Union
 
 from code_rag.api.models import ApiReport, SetupResult, SyncResult
+from code_rag.core.manager import CodeRAGManager
 from code_rag.core.models import KnowledgeUnit
 from code_rag.intelligence.distiller import DistillerConfig
 from code_rag.services.config import load_or_update_config
@@ -27,7 +28,7 @@ class CodeRAG:
         self._onnx = onnx
         self._root = Path(root) if root is not None else Path.cwd()
         self._allow_build_execution = allow_build_execution
-        self._manager = None
+        self._manager: Optional[CodeRAGManager] = None
 
     async def __aenter__(self) -> "CodeRAG":
         return self
@@ -40,22 +41,42 @@ class CodeRAG:
             await self._manager.close()
             self._manager = None
 
-    def _ensure_manager(self):
+    async def _ensure_manager(self, wipe: bool = False):
+        if wipe and self._manager is not None:
+            await self.close()
         if self._manager is None:
-            self._manager = create_manager(
-                self._db, self._onnx, allow_build_execution=self._allow_build_execution
+            self._manager = await create_manager(
+                self._db,
+                self._onnx,
+                allow_build_execution=self._allow_build_execution,
+                wipe=wipe,
             )
         return self._manager
 
-    async def config(
+    async def config(  # pylint: disable=too-many-arguments
         self,
         *,
         url: Optional[str] = None,
         key: Optional[str] = None,
         model: Optional[str] = None,
         provider: Optional[str] = None,
+        embedding_url: Optional[str] = None,
+        embedding_key: Optional[str] = None,
+        embedding_model: Optional[str] = None,
+        embedding_provider: Optional[str] = None,
+        clear_embedding: bool = False,
     ) -> DistillerConfig:
-        return load_or_update_config(url=url, key=key, model=model, provider=provider)
+        return load_or_update_config(
+            url=url,
+            key=key,
+            model=model,
+            provider=provider,
+            embedding_url=embedding_url,
+            embedding_key=embedding_key,
+            embedding_model=embedding_model,
+            embedding_provider=embedding_provider,
+            clear_embedding=clear_embedding,
+        )
 
     async def setup(self, *, force: bool = False) -> SetupResult:
         return await run_setup(force=force)
@@ -70,7 +91,7 @@ class CodeRAG:
         if path is None and not index_all:
             return SyncResult(status="success", indexed_files=0)
         return await run_sync(
-            self._ensure_manager(),
+            await self._ensure_manager(),
             root=self._root,
             path=path,
             index_all=index_all,
@@ -78,10 +99,12 @@ class CodeRAG:
         )
 
     async def search(self, query: str, *, limit: int = 5) -> list[KnowledgeUnit]:
-        return await run_search(self._ensure_manager(), query, limit=limit)
+        return await run_search(await self._ensure_manager(), query, limit=limit)
 
     async def api(self, library: str, *, lang: Optional[str] = None) -> ApiReport:
-        return await run_api(self._ensure_manager(), library, lang=lang)
+        return await run_api(await self._ensure_manager(), library, lang=lang)
 
     async def rebuild(self) -> SyncResult:
-        return await run_rebuild(self._ensure_manager(), root=self._root)
+        await self.close()
+        manager = await self._ensure_manager(wipe=True)
+        return await run_rebuild(manager, root=self._root)

{agent_coderag-1.3.3 → agent_coderag-1.3.4}/code_rag/api/models.py

@@ -1,9 +1,15 @@
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class SyncFileError(BaseModel):
+    file: str
+    message: str
 
 
 class SyncResult(BaseModel):
     status: str
     indexed_files: int
+    errors: list[SyncFileError] = Field(default_factory=list)
 
 
 class ApiReport(BaseModel):

{agent_coderag-1.3.3 → agent_coderag-1.3.4}/code_rag/core/constants.py

@@ -4,6 +4,10 @@ Central location for project-wide constants.
 
 # Embedding Model Constants
 EMBEDDING_DIM = 384
+EMBEDDING_BATCH_SIZE = 32
+EMBEDDING_REQUEST_TIMEOUT = 30
+EMBEDDING_PROBE_TEXT = "probe"
+LOCAL_EMBEDDING_MODEL_ID = "local:mini-lm"
 MAX_TOKEN_LENGTH = 512
 PAD_ID = 0
 PAD_TOKEN = "[PAD]"  # nosec B105

agent_coderag-1.3.4/code_rag/core/interfaces.py

@@ -0,0 +1,113 @@
+from abc import ABC, abstractmethod
+from typing import List, Optional
+from .models import KnowledgeUnit, Relation
+
+
+class IParser(ABC):
+    """Interface for extracting structure from code."""
+
+    @abstractmethod
+    async def distill_file(self, file_path: str) -> List[KnowledgeUnit]:
+        pass  # pragma: no cover
+
+
+class IEmbedder(ABC):
+    """Port for local ONNX and remote OpenAI-compatible embedders."""
+
+    @property
+    @abstractmethod
+    def dimension(self) -> int:
+        """Bound embedding width. No IO. Raise if unbound."""  # pragma: no cover
+
+    @property
+    @abstractmethod
+    def model_id(self) -> str:
+        """Stable id written to index_meta (local:mini-lm or remote model)."""  # pragma: no cover
+
+    @abstractmethod
+    def bind_dimension(self, dim: int) -> None:
+        """Record the index dimension before search/upsert."""  # pragma: no cover
+
+    @abstractmethod
+    async def aembed(self, texts: list[str]) -> list[list[float]]:
+        """Return len(texts) L2-normalized rows, 1:1 with input order."""  # pragma: no cover
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Release ONNX session and/or HTTP resources."""  # pragma: no cover
+
+
+class IStorage(ABC):
+    """Interface for storing the index."""
+
+    @abstractmethod
+    async def upsert_unit(
+        self, unit: KnowledgeUnit, vector: Optional[List[float]] = None
+    ):
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def has_embedding(self, unit_id: str) -> bool:
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def list_units(self) -> List[KnowledgeUnit]:
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def mark_embedding_model_synced(self) -> None:
+        pass  # pragma: no cover
+
+    @property
+    @abstractmethod
+    def embedding_model_dirty(self) -> bool:
+        pass  # pragma: no cover
+
+    @property
+    @abstractmethod
+    def embedder(self) -> IEmbedder:
+        """Bound embedder used for vector search and upsert."""  # pragma: no cover
+
+    @abstractmethod
+    async def get_unit(self, unit_id: str) -> Optional[KnowledgeUnit]:
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def search_units(self, query: str, limit: int = 5) -> List[KnowledgeUnit]:
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def upsert_relation(self, relation: Relation):
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def get_relations(
+        self, unit_id: str, direction: str = "out"
+    ) -> List[Relation]:
+        pass  # pragma: no cover
+
+    @abstractmethod
+    async def set_dependency_path(self, lib_name: str, path: str) -> None:
+        """Caches the absolute path to a library's JAR/binary."""
+
+    @abstractmethod
+    async def get_dependency_path(self, lib_name: str) -> Optional[str]:
+        """Retrieves the cached path for a library."""
+
+    @abstractmethod
+    async def delete_stale_units(
+        self, file_path: str, current_unit_ids: List[str]
+    ) -> None:
+        """Removes units that are no longer present in the given file."""
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Closes the storage connection and releases resources."""
+
+
+class IIntelligence(ABC):
+    """Interface for LLM-based analysis (distillation, embeddings)."""
+
+    @abstractmethod
+    async def summarize(self, code: str, unit_name: str) -> str:
+        pass  # pragma: no cover

{agent_coderag-1.3.3 → agent_coderag-1.3.4}/code_rag/core/manager.py

@@ -1,19 +1,34 @@
 import logging
 import asyncio
+import inspect
 import os
 import shutil
 import subprocess  # nosec
+import sys
 from pathlib import Path
 from typing import List, Optional
 from .interfaces import IStorage, IParser, IIntelligence
 from .models import KnowledgeUnit
 from .utils import validate_path
-from .constants import MAX_CONCURRENT_TASKS
+from .constants import MAX_CONCURRENT_TASKS, EMBEDDING_BATCH_SIZE
+from .exceptions import StorageError, IntelligenceError
 from ..discovery.manager import DiscoveryManager
 
 logger = logging.getLogger(__name__)
 
 
+def _report_worker_failure(path: str, exc: BaseException) -> tuple[str, str]:
+    logger.error("Worker failed to sync %s: %s", path, exc)
+    print(f"Worker failed to sync {path}: {exc}", file=sys.stderr, flush=True)
+    return (path, str(exc))
+
+
+def unit_embedding_text(unit: KnowledgeUnit) -> str:
+    return unit.summary or (
+        f"{unit.kind.value} {unit.name} {unit.signature or ''} {unit.docstring or ''}"
+    )
+
+
 class CodeRAGManager:
     """
     Orchestrates the RAG workflow: parsing, distillation, and storage.
@@ -192,20 +207,46 @@ allprojects {
             if init_script.exists():
                 init_script.unlink()
 
+    async def _reject_dirty_incremental(self) -> None:
+        if getattr(self.storage, "embedding_model_dirty", False):
+            raise StorageError("Embedding model changed; run rebuild or sync --all")
+
+    async def _ensure_embeddings(self) -> None:
+        ensure = getattr(self.storage, "ensure_embeddings_bound", None)
+        if callable(ensure):
+            result = ensure()
+            if inspect.isawaitable(result):
+                await result
+
+    async def _reembed_all_units(self) -> None:
+        units = await self.storage.list_units()
+        await self._embed_and_upsert(units)
+        await self.storage.mark_embedding_model_synced()
+
+    async def _embed_and_upsert(self, units: list[KnowledgeUnit]) -> None:
+        if not units:
+            return
+        await self._ensure_embeddings()
+        embedder = self.storage.embedder
+        for start in range(0, len(units), EMBEDDING_BATCH_SIZE):
+            chunk = units[start : start + EMBEDDING_BATCH_SIZE]
+            texts = [unit_embedding_text(unit) for unit in chunk]
+            vectors = await embedder.aembed(texts)
+            if len(vectors) != len(chunk):
+                raise IntelligenceError("Embedding count mismatch")
+            for unit, vector in zip(chunk, vectors):
+                await self.storage.upsert_unit(unit, vector=vector)
+
     async def sync_file(self, file_path: str, force_distill: bool = False) -> None:
         """
         Processes a single file and syncs it with the storage.
         """
-        # 1. Parse AST to get units
+        await self._reject_dirty_incremental()
         current_units = await self.parser.distill_file(file_path)
-
+        pending: list[KnowledgeUnit] = []
         for unit in current_units:
-            # v5.40: Delta-distillation logic
             raw_code = unit.metadata.pop("raw_code", "")
-
-            # 2. Get existing unit to check hash
             existing_unit = await self.storage.get_unit(unit.id)
-
             should_distill = force_distill
             if not existing_unit:
                 should_distill = True
@@ -216,29 +257,36 @@ allprojects {
             elif not existing_unit.summary:
                 should_distill = True
                 logger.info("Summary missing for %s", unit.name)
-
             if should_distill:
                 async with self.semaphore:
                     logger.info(
                         "Distilling summary for %s in %s...", unit.name, unit.path
                     )
                     try:
-                        summary = await self.intelligence.summarize(raw_code, unit.name)
-                        unit.summary = summary
-                    except Exception as e:
-                        logger.error("Failed to distill %s: %s", unit.name, e)
-                        # Keep old summary if available, otherwise stay None
+                        unit.summary = await self.intelligence.summarize(
+                            raw_code, unit.name
+                        )
+                    except Exception as exc:
+                        logger.error("Failed to distill %s: %s", unit.name, exc)
                         unit.summary = existing_unit.summary if existing_unit else None
             else:
-                # Reuse existing summary if code hasn't changed
                 unit.summary = existing_unit.summary if existing_unit else None
 
-            # 3. Save to storage (includes embedding generation)
-            await self.storage.upsert_unit(unit)
-
-        # 4. Garbage Collection: Remove units that were in this file but are no longer there
-        current_unit_ids = [u.id for u in current_units]
-        await self.storage.delete_stale_units(file_path, current_unit_ids)
+            has_vec = await self.storage.has_embedding(unit.id)
+            skip = (
+                existing_unit is not None
+                and existing_unit.code_hash == unit.code_hash
+                and existing_unit.summary == unit.summary
+                and has_vec
+            )
+            if skip:
+                await self.storage.upsert_unit(unit)
+            else:
+                pending.append(unit)
+        await self._embed_and_upsert(pending)
+        await self.storage.delete_stale_units(
+            file_path, [unit.id for unit in current_units]
+        )
 
     async def search(self, query: str, limit: int = 5) -> List[KnowledgeUnit]:
         """
@@ -246,38 +294,58 @@ allprojects {
         """
         return await self.storage.search_units(query, limit=limit)
 
-    async def sync_project(self, paths: List[str], force_distill: bool = False) -> None:
+    async def sync_project(
+        self,
+        paths: List[str],
+        force_distill: bool = False,
+        *,
+        index_all: bool = False,
+    ) -> list[tuple[str, str]]:
         """
         Concurrent synchronization of multiple files using a worker pool.
+
+        Returns (path, error) pairs for files that failed. The queue is drained
+        even if some workers fail; callers must not treat an empty return as
+        "raised" — check the list instead of catching after completion.
         """
+        if getattr(self.storage, "embedding_model_dirty", False):
+            if not index_all:
+                raise StorageError("Embedding model changed; run rebuild or sync --all")
+            await self._reembed_all_units()
         if not paths:
-            return
+            return []
 
         queue: asyncio.Queue[str] = asyncio.Queue()
-        for p in paths:
-            await queue.put(p)
+        for path in paths:
+            await queue.put(path)
 
-        async def worker() -> None:
+        async def worker() -> list[tuple[str, str]]:
+            failures: list[tuple[str, str]] = []
             while not queue.empty():
                 path = await queue.get()
                 try:
-                    await self.sync_file(path, force_distill=force_distill)
-                except Exception as e:
-                    logger.error("Worker failed to sync %s: %s", path, e)
+                    await self.sync_file(
+                        path,
+                        force_distill=force_distill,
+                    )
+                except Exception as exc:
+                    failures.append(_report_worker_failure(path, exc))
                 finally:
                     queue.task_done()
+            return failures
 
-        # Run limited number of workers
         worker_count = min(len(paths), self.max_concurrency)
         tasks = [asyncio.create_task(worker()) for _ in range(worker_count)]
-
-        await asyncio.gather(*tasks)
+        batches = await asyncio.gather(*tasks)
         logger.info("Project sync complete.")
+        return [item for batch in batches for item in batch]
 
     async def close(self) -> None:
         """Releases manager resources."""
         await self.storage.close()
-        # If intelligence has close method (Embedder does)
         if hasattr(self.intelligence, "close"):
-            self.intelligence.close()
+            closer = self.intelligence.close
+            result = closer()
+            if asyncio.iscoroutine(result):
+                await result
         logger.info("CodeRAG manager closed.")