kodit 0.4.2__py3-none-any.whl → 0.5.0__py3-none-any.whl

kodit/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.4.2'
-__version_tuple__ = version_tuple = (0, 4, 2)
+__version__ = version = '0.5.0'
+__version_tuple__ = version_tuple = (0, 5, 0)
 
 __commit_id__ = commit_id = None

kodit/app.py

@@ -3,70 +3,105 @@
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 
+import structlog
 from asgi_correlation_id import CorrelationIdMiddleware
 from fastapi import FastAPI, Response
 from fastapi.responses import RedirectResponse
 
 from kodit._version import version
 from kodit.application.factories.reporting_factory import create_server_operation
-from kodit.application.services.auto_indexing_service import AutoIndexingService
+from kodit.application.factories.server_factory import ServerFactory
 from kodit.application.services.indexing_worker_service import IndexingWorkerService
 from kodit.application.services.sync_scheduler import SyncSchedulerService
 from kodit.config import AppContext
-from kodit.infrastructure.api.v1.routers import (
-    indexes_router,
-    queue_router,
-    search_router,
+from kodit.domain.value_objects import (
+    Document,
+    EnrichmentIndexRequest,
+    EnrichmentRequest,
+    IndexRequest,
 )
+from kodit.infrastructure.api.v1.routers.commits import router as commits_router
+from kodit.infrastructure.api.v1.routers.queue import router as queue_router
+from kodit.infrastructure.api.v1.routers.repositories import (
+    router as repositories_router,
+)
+from kodit.infrastructure.api.v1.routers.search import router as search_router
 from kodit.infrastructure.api.v1.schemas.context import AppLifespanState
+from kodit.infrastructure.sqlalchemy.task_status_repository import (
+    create_task_status_repository,
+)
 from kodit.mcp import mcp
-from kodit.middleware import ASGICancelledErrorMiddleware, logging_middleware
+from kodit.middleware import (
+    ASGICancelledErrorMiddleware,
+    logging_middleware,
+)
 
 # Global services
-_auto_indexing_service: AutoIndexingService | None = None
 _sync_scheduler_service: SyncSchedulerService | None = None
+_server_factory: ServerFactory | None = None
 
 
 @asynccontextmanager
 async def app_lifespan(_: FastAPI) -> AsyncIterator[AppLifespanState]:
     """Manage application lifespan for auto-indexing and sync."""
-    global _auto_indexing_service, _sync_scheduler_service  # noqa: PLW0603
+    global _sync_scheduler_service  # noqa: PLW0603
+    global _server_factory  # noqa: PLW0603
 
     # App context has already been configured by the CLI.
     app_context = AppContext()
     db = await app_context.get_db()
-    operation = create_server_operation()
+    log = structlog.get_logger(__name__)
+    operation = create_server_operation(
+        create_task_status_repository(db.session_factory)
+    )
+
+    _server_factory = ServerFactory(app_context, db.session_factory)
+
+    # Quickly check if the providers are accessible and raise an error if not
+    log.info("Checking providers are accessible")
+    try:
+        await anext(
+            _server_factory.code_search_service().index_documents(
+                IndexRequest(
+                    documents=[Document(snippet_id="1", text="def hello(): pass")]
+                )
+            )
+        )
+    except Exception as e:
+        raise ValueError("Embedding service is not accessible") from e
+    try:
+        await anext(
+            _server_factory.enrichment_service().enrich_documents(
+                EnrichmentIndexRequest(
+                    requests=[
+                        EnrichmentRequest(snippet_id="1", text="def hello(): pass")
+                    ]
+                )
+            )
+        )
+    except Exception as e:
+        raise ValueError("Enrichment service is not accessible") from e
 
     # Start the queue worker service
     _indexing_worker_service = IndexingWorkerService(
         app_context=app_context,
         session_factory=db.session_factory,
+        server_factory=_server_factory,
     )
     await _indexing_worker_service.start(operation)
 
-    # Start auto-indexing service
-    _auto_indexing_service = AutoIndexingService(
-        app_context=app_context,
-        session_factory=db.session_factory,
-    )
-    await _auto_indexing_service.start_background_indexing(operation)
-
     # Start sync scheduler service
     if app_context.periodic_sync.enabled:
-        _sync_scheduler_service = SyncSchedulerService(
-            session_factory=db.session_factory,
-        )
+        _sync_scheduler_service = _server_factory.sync_scheduler_service()
         _sync_scheduler_service.start_periodic_sync(
             interval_seconds=app_context.periodic_sync.interval_seconds
         )
 
-    yield AppLifespanState(app_context=app_context)
+    yield AppLifespanState(app_context=app_context, server_factory=_server_factory)
 
     # Stop services
     if _sync_scheduler_service:
         await _sync_scheduler_service.stop_periodic_sync()
-    if _auto_indexing_service:
-        await _auto_indexing_service.stop()
     if _indexing_worker_service:
         await _indexing_worker_service.stop()
 
@@ -118,10 +153,10 @@ async def healthz() -> Response:
 
 
 # Include API routers
-app.include_router(indexes_router)
 app.include_router(queue_router)
 app.include_router(search_router)
-
+app.include_router(commits_router)
+app.include_router(repositories_router)
 
 # Add mcp routes last, otherwise previous routes aren't added
 # Mount both apps at root - they have different internal paths

kodit/application/factories/reporting_factory.py

@@ -1,27 +1,36 @@
 """Reporting factory."""
 
-from kodit.application.services.reporting import OperationType, ProgressTracker
+from kodit.application.services.reporting import ProgressTracker, TaskOperation
 from kodit.config import ReportingConfig
+from kodit.domain.protocols import TaskStatusRepository
+from kodit.infrastructure.reporting.db_progress import DBProgressReportingModule
 from kodit.infrastructure.reporting.log_progress import LoggingReportingModule
-from kodit.infrastructure.reporting.tdqm_progress import TQDMReportingModule
+from kodit.infrastructure.reporting.telemetry_progress import (
+    TelemetryProgressReportingModule,
+)
 
 
 def create_noop_operation() -> ProgressTracker:
     """Create a noop reporter."""
-    return ProgressTracker(OperationType.ROOT.value)
+    return ProgressTracker.create(TaskOperation.ROOT)
 
 
 def create_cli_operation(config: ReportingConfig | None = None) -> ProgressTracker:
     """Create a CLI reporter."""
     shared_config = config or ReportingConfig()
-    s = ProgressTracker(OperationType.ROOT.value)
-    s.subscribe(TQDMReportingModule(shared_config))
+    s = ProgressTracker.create(TaskOperation.ROOT)
+    s.subscribe(TelemetryProgressReportingModule())
+    s.subscribe(LoggingReportingModule(shared_config))
     return s
 
 
-def create_server_operation(config: ReportingConfig | None = None) -> ProgressTracker:
+def create_server_operation(
+    task_status_repository: TaskStatusRepository, config: ReportingConfig | None = None
+) -> ProgressTracker:
     """Create a server reporter."""
     shared_config = config or ReportingConfig()
-    s = ProgressTracker(OperationType.ROOT.value)
+    s = ProgressTracker.create(TaskOperation.ROOT)
+    s.subscribe(TelemetryProgressReportingModule())
     s.subscribe(LoggingReportingModule(shared_config))
+    s.subscribe(DBProgressReportingModule(task_status_repository, shared_config))
     return s

kodit/application/factories/server_factory.py

@@ -0,0 +1,311 @@
+"""Create a big object that contains all the application services."""
+
+from collections.abc import Callable
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.application.factories.reporting_factory import create_server_operation
+from kodit.application.services.code_search_application_service import (
+    CodeSearchApplicationService,
+)
+from kodit.application.services.commit_indexing_application_service import (
+    CommitIndexingApplicationService,
+)
+from kodit.application.services.queue_service import QueueService
+from kodit.application.services.reporting import ProgressTracker
+from kodit.application.services.sync_scheduler import SyncSchedulerService
+from kodit.config import AppContext
+from kodit.domain.protocols import (
+    FusionService,
+    GitAdapter,
+    GitBranchRepository,
+    GitCommitRepository,
+    GitRepoRepository,
+    GitTagRepository,
+    SnippetRepositoryV2,
+    TaskStatusRepository,
+)
+from kodit.domain.services.bm25_service import BM25DomainService, BM25Repository
+from kodit.domain.services.embedding_service import EmbeddingDomainService
+from kodit.domain.services.enrichment_service import EnrichmentDomainService
+from kodit.domain.services.git_repository_service import (
+    GitRepositoryScanner,
+    RepositoryCloner,
+)
+from kodit.infrastructure.bm25.local_bm25_repository import LocalBM25Repository
+from kodit.infrastructure.bm25.vectorchord_bm25_repository import (
+    VectorChordBM25Repository,
+)
+from kodit.infrastructure.cloning.git.git_python_adaptor import GitPythonAdapter
+from kodit.infrastructure.embedding.embedding_factory import (
+    embedding_domain_service_factory,
+)
+from kodit.infrastructure.enrichment.enrichment_factory import (
+    enrichment_domain_service_factory,
+)
+
+# InMemoryGitTagRepository removed - now handled by InMemoryGitRepoRepository
+from kodit.infrastructure.indexing.fusion_service import ReciprocalRankFusionService
+from kodit.infrastructure.slicing.slicer import Slicer
+from kodit.infrastructure.sqlalchemy.embedding_repository import (
+    SqlAlchemyEmbeddingRepository,
+    create_embedding_repository,
+)
+from kodit.infrastructure.sqlalchemy.git_branch_repository import (
+    create_git_branch_repository,
+)
+from kodit.infrastructure.sqlalchemy.git_commit_repository import (
+    create_git_commit_repository,
+)
+from kodit.infrastructure.sqlalchemy.git_repository import create_git_repo_repository
+from kodit.infrastructure.sqlalchemy.git_tag_repository import (
+    create_git_tag_repository,
+)
+from kodit.infrastructure.sqlalchemy.snippet_v2_repository import (
+    create_snippet_v2_repository,
+)
+from kodit.infrastructure.sqlalchemy.task_status_repository import (
+    create_task_status_repository,
+)
+from kodit.infrastructure.sqlalchemy.unit_of_work import SqlAlchemyUnitOfWork
+
+
+class ServerFactory:
+    """Factory for creating server application services."""
+
+    def __init__(
+        self,
+        app_context: AppContext,
+        session_factory: Callable[[], AsyncSession],
+    ) -> None:
+        """Initialize the ServerFactory."""
+        self.app_context = app_context
+        self.session_factory = session_factory
+        self._repo_repository: GitRepoRepository | None = None
+        self._snippet_v2_repository: SnippetRepositoryV2 | None = None
+        self._git_adapter: GitAdapter | None = None
+        self._scanner: GitRepositoryScanner | None = None
+        self._cloner: RepositoryCloner | None = None
+        self._commit_indexing_application_service: (
+            CommitIndexingApplicationService | None
+        ) = None
+        self._enrichment_service: EnrichmentDomainService | None = None
+        self._task_status_repository: TaskStatusRepository | None = None
+        self._operation: ProgressTracker | None = None
+        self._queue_service: QueueService | None = None
+        self._slicer: Slicer | None = None
+        self._bm25_service: BM25DomainService | None = None
+        self._bm25_repository: BM25Repository | None = None
+        self._code_search_service: EmbeddingDomainService | None = None
+        self._text_search_service: EmbeddingDomainService | None = None
+        self._sync_scheduler_service: SyncSchedulerService | None = None
+        self._embedding_repository: SqlAlchemyEmbeddingRepository | None = None
+        self._fusion_service: FusionService | None = None
+        self._code_search_application_service: CodeSearchApplicationService | None = (
+            None
+        )
+        self._git_commit_repository: GitCommitRepository | None = None
+        self._git_branch_repository: GitBranchRepository | None = None
+        self._git_tag_repository: GitTagRepository | None = None
+
+    def queue_service(self) -> QueueService:
+        """Create a QueueService instance."""
+        if not self._queue_service:
+            self._queue_service = QueueService(session_factory=self.session_factory)
+        return self._queue_service
+
+    def task_status_repository(self) -> TaskStatusRepository:
+        """Create a TaskStatusRepository instance."""
+        if not self._task_status_repository:
+            self._task_status_repository = create_task_status_repository(
+                session_factory=self.session_factory
+            )
+        return self._task_status_repository
+
+    def operation(self) -> ProgressTracker:
+        """Create a ProgressTracker instance."""
+        if not self._operation:
+            self._operation = create_server_operation(
+                task_status_repository=self.task_status_repository()
+            )
+        return self._operation
+
+    def slicer(self) -> Slicer:
+        """Create a Slicer instance."""
+        if not self._slicer:
+            self._slicer = Slicer()
+        return self._slicer
+
+    def bm25_repository(self) -> BM25Repository:
+        """Create a BM25Repository instance."""
+        if not self._bm25_repository:
+            if self.app_context.default_search.provider == "vectorchord":
+                self._bm25_repository = VectorChordBM25Repository(
+                    session_factory=self.session_factory
+                )
+            else:
+                self._bm25_repository = LocalBM25Repository(
+                    data_dir=self.app_context.get_data_dir()
+                )
+        return self._bm25_repository
+
+    def bm25_service(self) -> BM25DomainService:
+        """Create a BM25DomainService instance."""
+        if not self._bm25_service:
+            self._bm25_service = BM25DomainService(repository=self.bm25_repository())
+        return self._bm25_service
+
+    def code_search_service(self) -> EmbeddingDomainService:
+        """Create a EmbeddingDomainService instance."""
+        if not self._code_search_service:
+            self._code_search_service = embedding_domain_service_factory(
+                "code", self.app_context, self.session_factory
+            )
+        return self._code_search_service
+
+    def text_search_service(self) -> EmbeddingDomainService:
+        """Create a EmbeddingDomainService instance."""
+        if not self._text_search_service:
+            self._text_search_service = embedding_domain_service_factory(
+                "text", self.app_context, self.session_factory
+            )
+        return self._text_search_service
+
+    def commit_indexing_application_service(self) -> CommitIndexingApplicationService:
+        """Create a CommitIndexingApplicationService instance."""
+        if not self._commit_indexing_application_service:
+            self._commit_indexing_application_service = (
+                CommitIndexingApplicationService(
+                    snippet_v2_repository=self.snippet_v2_repository(),
+                    repo_repository=self.repo_repository(),
+                    git_commit_repository=self.git_commit_repository(),
+                    git_branch_repository=self.git_branch_repository(),
+                    git_tag_repository=self.git_tag_repository(),
+                    operation=self.operation(),
+                    scanner=self.scanner(),
+                    cloner=self.cloner(),
+                    snippet_repository=self.snippet_v2_repository(),
+                    slicer=self.slicer(),
+                    queue=self.queue_service(),
+                    bm25_service=self.bm25_service(),
+                    code_search_service=self.code_search_service(),
+                    text_search_service=self.text_search_service(),
+                    enrichment_service=self.enrichment_service(),
+                    embedding_repository=self.embedding_repository(),
+                )
+            )
+
+        return self._commit_indexing_application_service
+
+    def unit_of_work(self) -> SqlAlchemyUnitOfWork:
+        """Create a SqlAlchemyUnitOfWork instance."""
+        return SqlAlchemyUnitOfWork(session_factory=self.session_factory)
+
+    def repo_repository(self) -> GitRepoRepository:
+        """Create a GitRepoRepository instance."""
+        if not self._repo_repository:
+            self._repo_repository = create_git_repo_repository(
+                session_factory=self.session_factory
+            )
+        return self._repo_repository
+
+    # branch_repository and commit_repository removed - now handled by repo_repository
+    # as GitRepo is the aggregate root
+
+    def git_adapter(self) -> GitAdapter:
+        """Create a GitAdapter instance."""
+        if not self._git_adapter:
+            self._git_adapter = GitPythonAdapter()
+        return self._git_adapter
+
+    # tag_repository removed - now handled by repo_repository
+
+    def scanner(self) -> GitRepositoryScanner:
+        """Create a GitRepositoryScanner instance."""
+        if not self._scanner:
+            self._scanner = GitRepositoryScanner(self.git_adapter())
+        return self._scanner
+
+    def cloner(self) -> RepositoryCloner:
+        """Create a RepositoryCloner instance."""
+        if not self._cloner:
+            self._cloner = RepositoryCloner(
+                self.git_adapter(), self.app_context.get_clone_dir()
+            )
+        return self._cloner
+
+    def snippet_v2_repository(self) -> SnippetRepositoryV2:
+        """Create a SnippetRepositoryV2 instance."""
+        if not self._snippet_v2_repository:
+            self._snippet_v2_repository = create_snippet_v2_repository(
+                session_factory=self.session_factory
+            )
+        return self._snippet_v2_repository
+
+    def enrichment_service(self) -> EnrichmentDomainService:
+        """Create a EnrichmentDomainService instance."""
+        if not self._enrichment_service:
+            self._enrichment_service = enrichment_domain_service_factory(
+                self.app_context
+            )
+        return self._enrichment_service
+
+    def sync_scheduler_service(self) -> SyncSchedulerService:
+        """Create a SyncSchedulerService instance."""
+        if not self._sync_scheduler_service:
+            self._sync_scheduler_service = SyncSchedulerService(
+                queue_service=self.queue_service(),
+                repo_repository=self.repo_repository(),
+            )
+        return self._sync_scheduler_service
+
+    def embedding_repository(self) -> SqlAlchemyEmbeddingRepository:
+        """Create a SqlAlchemyEmbeddingRepository instance."""
+        if not self._embedding_repository:
+            self._embedding_repository = create_embedding_repository(
+                session_factory=self.session_factory
+            )
+        return self._embedding_repository
+
+    def fusion_service(self) -> FusionService:
+        """Create a FusionService instance."""
+        if not self._fusion_service:
+            self._fusion_service = ReciprocalRankFusionService()
+        return self._fusion_service
+
+    def code_search_application_service(self) -> CodeSearchApplicationService:
+        """Create a CodeSearchApplicationService instance."""
+        if not self._code_search_application_service:
+            self._code_search_application_service = CodeSearchApplicationService(
+                bm25_service=self.bm25_service(),
+                code_search_service=self.code_search_service(),
+                text_search_service=self.text_search_service(),
+                progress_tracker=self.operation(),
+                snippet_repository=self.snippet_v2_repository(),
+                fusion_service=self.fusion_service(),
+            )
+        return self._code_search_application_service
+
+    def git_commit_repository(self) -> GitCommitRepository:
+        """Create a GitCommitRepository instance."""
+        if not self._git_commit_repository:
+            self._git_commit_repository = create_git_commit_repository(
+                session_factory=self.session_factory
+            )
+        return self._git_commit_repository
+
+    def git_branch_repository(self) -> GitBranchRepository:
+        """Create a GitBranchRepository instance."""
+        if not self._git_branch_repository:
+            self._git_branch_repository = create_git_branch_repository(
+                session_factory=self.session_factory
+            )
+        return self._git_branch_repository
+
+    def git_tag_repository(self) -> GitTagRepository:
+        """Create a GitTagRepository instance."""
+        if not self._git_tag_repository:
+            self._git_tag_repository = create_git_tag_repository(
+                session_factory=self.session_factory
+            )
+        return self._git_tag_repository

kodit/application/services/code_search_application_service.py

@@ -0,0 +1,144 @@
+"""Service for searching the indexes."""
+
+from dataclasses import dataclass
+
+import structlog
+
+from kodit.application.services.reporting import ProgressTracker
+from kodit.domain.entities.git import SnippetV2
+from kodit.domain.protocols import FusionService, SnippetRepositoryV2
+from kodit.domain.services.bm25_service import BM25DomainService
+from kodit.domain.services.embedding_service import EmbeddingDomainService
+from kodit.domain.value_objects import (
+    FusionRequest,
+    MultiSearchRequest,
+    SearchRequest,
+    SearchResult,
+)
+from kodit.log import log_event
+
+
+@dataclass
+class MultiSearchResult:
+    """Enhanced search result with comprehensive snippet metadata."""
+
+    snippet: SnippetV2
+    original_scores: list[float]
+
+    def to_json(self) -> str:
+        """Return LLM-optimized JSON representation following the compact schema."""
+        return self.snippet.model_dump_json()
+
+    @classmethod
+    def to_jsonlines(cls, results: list["MultiSearchResult"]) -> str:
+        """Convert multiple MultiSearchResult objects to JSON Lines format.
+
+        Args:
+            results: List of MultiSearchResult objects
+            include_summary: Whether to include summary fields
+
+        Returns:
+            JSON Lines string (one JSON object per line)
+
+        """
+        return "\n".join(result.to_json() for result in results)
+
+
+class CodeSearchApplicationService:
+    """Service for searching the indexes."""
+
+    def __init__(  # noqa: PLR0913
+        self,
+        bm25_service: BM25DomainService,
+        code_search_service: EmbeddingDomainService,
+        text_search_service: EmbeddingDomainService,
+        progress_tracker: ProgressTracker,
+        snippet_repository: SnippetRepositoryV2,
+        fusion_service: FusionService,
+    ) -> None:
+        """Initialize the code search application service."""
+        self.bm25_service = bm25_service
+        self.code_search_service = code_search_service
+        self.text_search_service = text_search_service
+        self.progress_tracker = progress_tracker
+        self.snippet_repository = snippet_repository
+        self.fusion_service = fusion_service
+        self.log = structlog.get_logger(__name__)
+
+    async def search(self, request: MultiSearchRequest) -> list[MultiSearchResult]:
+        """Search for relevant snippets across all indexes."""
+        log_event("kodit.index.search")
+
+        # Apply filters if provided
+        filtered_snippet_ids: list[str] | None = None
+        # TODO(Phil): Re-implement filtering on search results
+
+        # Gather results from different search modes
+        fusion_list: list[list[FusionRequest]] = []
+
+        # Keyword search
+        if request.keywords:
+            result_ids: list[SearchResult] = []
+            for keyword in request.keywords:
+                results = await self.bm25_service.search(
+                    SearchRequest(
+                        query=keyword,
+                        top_k=request.top_k,
+                        snippet_ids=filtered_snippet_ids,
+                    )
+                )
+                result_ids.extend(results)
+
+            fusion_list.append(
+                [FusionRequest(id=x.snippet_id, score=x.score) for x in result_ids]
+            )
+
+        # Semantic code search
+        if request.code_query:
+            query_results = await self.code_search_service.search(
+                SearchRequest(
+                    query=request.code_query,
+                    top_k=request.top_k,
+                    snippet_ids=filtered_snippet_ids,
+                )
+            )
+            fusion_list.append(
+                [FusionRequest(id=x.snippet_id, score=x.score) for x in query_results]
+            )
+
+        # Semantic text search
+        if request.text_query:
+            query_results = await self.text_search_service.search(
+                SearchRequest(
+                    query=request.text_query,
+                    top_k=request.top_k,
+                    snippet_ids=filtered_snippet_ids,
+                )
+            )
+            fusion_list.append(
+                [FusionRequest(id=x.snippet_id, score=x.score) for x in query_results]
+            )
+
+        if len(fusion_list) == 0:
+            return []
+
+        # Fusion ranking
+        final_results = self.fusion_service.reciprocal_rank_fusion(
+            rankings=fusion_list,
+            k=60,  # This is a parameter in the RRF algorithm, not top_k
+        )
+
+        # Keep only top_k results
+        final_results = final_results[: request.top_k]
+
+        # Get snippet details
+        ids = [x.id for x in final_results]
+        search_results = await self.snippet_repository.get_by_ids(ids)
+        search_results.sort(key=lambda x: ids.index(x.id))
+        return [
+            MultiSearchResult(
+                snippet=snippet,
+                original_scores=[x.score for x in final_results if x.id == snippet.id],
+            )
+            for snippet in search_results
+        ]