kodit 0.4.1__tar.gz → 0.4.2__tar.gz

{kodit-0.4.1 → kodit-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.4.1
+Version: 0.4.2
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/

{kodit-0.4.1 → kodit-0.4.2}/docs/reference/api/index.md

@@ -12,7 +12,7 @@ look at the [hosted version](https://kodit.helix.ml/docs).
 This is the REST API for the Kodit server. Please refer to the
 [Kodit documentation](https://docs.helix.ml/kodit/) for more information.
     
-Current version: 0.3.18
+Current version: 0.4.2
 
 ## Authentication
 

{kodit-0.4.1 → kodit-0.4.2}/docs/reference/api/openapi.json

@@ -3,7 +3,7 @@
   "info": {
     "title": "kodit API",
     "description": "\nThis is the REST API for the Kodit server. Please refer to the\n[Kodit documentation](https://docs.helix.ml/kodit/) for more information.\n    ",
-    "version": "0.3.18"
+    "version": "0.4.2"
   },
   "paths": {
     "/healthz": {

{kodit-0.4.1 → kodit-0.4.2}/src/kodit/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.4.1'
-__version_tuple__ = version_tuple = (0, 4, 1)
+__version__ = version = '0.4.2'
+__version_tuple__ = version_tuple = (0, 4, 2)
 
 __commit_id__ = commit_id = None

{kodit-0.4.1 → kodit-0.4.2}/src/kodit/app.py

@@ -8,6 +8,7 @@ from fastapi import FastAPI, Response
 from fastapi.responses import RedirectResponse
 
 from kodit._version import version
+from kodit.application.factories.reporting_factory import create_server_operation
 from kodit.application.services.auto_indexing_service import AutoIndexingService
 from kodit.application.services.indexing_worker_service import IndexingWorkerService
 from kodit.application.services.sync_scheduler import SyncSchedulerService
@@ -34,20 +35,21 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[AppLifespanState]:
     # App context has already been configured by the CLI.
     app_context = AppContext()
     db = await app_context.get_db()
+    operation = create_server_operation()
 
     # Start the queue worker service
     _indexing_worker_service = IndexingWorkerService(
         app_context=app_context,
         session_factory=db.session_factory,
     )
-    await _indexing_worker_service.start()
+    await _indexing_worker_service.start(operation)
 
     # Start auto-indexing service
     _auto_indexing_service = AutoIndexingService(
         app_context=app_context,
         session_factory=db.session_factory,
     )
-    await _auto_indexing_service.start_background_indexing()
+    await _auto_indexing_service.start_background_indexing(operation)
 
     # Start sync scheduler service
     if app_context.periodic_sync.enabled:

{kodit-0.4.1 → kodit-0.4.2}/src/kodit/application/factories/code_indexing_factory.py

@@ -1,10 +1,20 @@
 """Factory for creating the unified code indexing application service."""
 
+from collections.abc import Callable
+
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from kodit.application.factories.reporting_factory import (
+    create_cli_operation,
+    create_noop_operation,
+    create_server_operation,
+)
 from kodit.application.services.code_indexing_application_service import (
     CodeIndexingApplicationService,
 )
+from kodit.application.services.reporting import (
+    ProgressTracker,
+)
 from kodit.config import AppContext
 from kodit.domain.services.bm25_service import BM25DomainService
 from kodit.domain.services.embedding_service import EmbeddingDomainService
@@ -35,23 +45,31 @@ from kodit.infrastructure.slicing.language_detection_service import (
     FileSystemLanguageDetectionService,
 )
 from kodit.infrastructure.sqlalchemy.embedding_repository import (
-    SqlAlchemyEmbeddingRepository,
+    create_embedding_repository,
 )
 from kodit.infrastructure.sqlalchemy.entities import EmbeddingType
-from kodit.infrastructure.sqlalchemy.index_repository import SqlAlchemyIndexRepository
+from kodit.infrastructure.sqlalchemy.index_repository import (
+    create_index_repository,
+)
 
 
 def create_code_indexing_application_service(
     app_context: AppContext,
     session: AsyncSession,
+    session_factory: Callable[[], AsyncSession],
+    operation: ProgressTracker,
 ) -> CodeIndexingApplicationService:
     """Create a unified code indexing application service with all dependencies."""
     # Create domain services
     bm25_service = BM25DomainService(bm25_repository_factory(app_context, session))
-    code_search_service = embedding_domain_service_factory("code", app_context, session)
-    text_search_service = embedding_domain_service_factory("text", app_context, session)
+    code_search_service = embedding_domain_service_factory(
+        "code", app_context, session, session_factory
+    )
+    text_search_service = embedding_domain_service_factory(
+        "text", app_context, session, session_factory
+    )
     enrichment_service = enrichment_domain_service_factory(app_context)
-    index_repository = SqlAlchemyIndexRepository(session=session)
+    index_repository = create_index_repository(session_factory=session_factory)
     # Use the unified language mapping from the domain layer
     language_map = LanguageMapping.get_extension_to_language_map()
 
@@ -78,17 +96,45 @@ def create_code_indexing_application_service(
         text_search_service=text_search_service,
         enrichment_service=enrichment_service,
         session=session,
+        operation=operation,
+    )
+
+
+def create_cli_code_indexing_application_service(
+    app_context: AppContext,
+    session: AsyncSession,
+    session_factory: Callable[[], AsyncSession],
+) -> CodeIndexingApplicationService:
+    """Create a CLI code indexing application service."""
+    return create_code_indexing_application_service(
+        app_context,
+        session,
+        session_factory,
+        create_cli_operation(),
+    )
+
+
+def create_server_code_indexing_application_service(
+    app_context: AppContext,
+    session: AsyncSession,
+    session_factory: Callable[[], AsyncSession],
+) -> CodeIndexingApplicationService:
+    """Create a server code indexing application service."""
+    return create_code_indexing_application_service(
+        app_context, session, session_factory, create_server_operation()
     )
 
 
 def create_fast_test_code_indexing_application_service(
     app_context: AppContext,
     session: AsyncSession,
+    session_factory: Callable[[], AsyncSession],
 ) -> CodeIndexingApplicationService:
     """Create a fast test code indexing application service."""
     # Create domain services
     bm25_service = BM25DomainService(bm25_repository_factory(app_context, session))
-    embedding_repository = SqlAlchemyEmbeddingRepository(session=session)
+    embedding_repository = create_embedding_repository(session_factory=session_factory)
+    operation = create_noop_operation()
 
     code_search_repository = LocalVectorSearchRepository(
         embedding_repository=embedding_repository,
@@ -116,7 +162,7 @@ def create_fast_test_code_indexing_application_service(
         enrichment_provider=NullEnrichmentProvider()
     )
 
-    index_repository = SqlAlchemyIndexRepository(session=session)
+    index_repository = create_index_repository(session_factory=session_factory)
     # Use the unified language mapping from the domain layer
     language_map = LanguageMapping.get_extension_to_language_map()
 
@@ -143,4 +189,5 @@ def create_fast_test_code_indexing_application_service(
         text_search_service=text_search_service,
         enrichment_service=enrichment_service,
         session=session,
+        operation=operation,
     )

kodit-0.4.2/src/kodit/application/factories/reporting_factory.py

@@ -0,0 +1,27 @@
+"""Reporting factory."""
+
+from kodit.application.services.reporting import OperationType, ProgressTracker
+from kodit.config import ReportingConfig
+from kodit.infrastructure.reporting.log_progress import LoggingReportingModule
+from kodit.infrastructure.reporting.tdqm_progress import TQDMReportingModule
+
+
+def create_noop_operation() -> ProgressTracker:
+    """Create a noop reporter."""
+    return ProgressTracker(OperationType.ROOT.value)
+
+
+def create_cli_operation(config: ReportingConfig | None = None) -> ProgressTracker:
+    """Create a CLI reporter."""
+    shared_config = config or ReportingConfig()
+    s = ProgressTracker(OperationType.ROOT.value)
+    s.subscribe(TQDMReportingModule(shared_config))
+    return s
+
+
+def create_server_operation(config: ReportingConfig | None = None) -> ProgressTracker:
+    """Create a server reporter."""
+    shared_config = config or ReportingConfig()
+    s = ProgressTracker(OperationType.ROOT.value)
+    s.subscribe(LoggingReportingModule(shared_config))
+    return s

{kodit-0.4.1 → kodit-0.4.2}/src/kodit/application/services/auto_indexing_service.py

@@ -11,7 +11,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from kodit.application.factories.code_indexing_factory import (
     create_code_indexing_application_service,
 )
+from kodit.application.factories.reporting_factory import create_noop_operation
 from kodit.application.services.queue_service import QueueService
+from kodit.application.services.reporting import ProgressTracker
 from kodit.config import AppContext
 from kodit.domain.entities import Task
 from kodit.domain.value_objects import QueuePriority
@@ -31,8 +33,11 @@ class AutoIndexingService:
         self.log = structlog.get_logger(__name__)
         self._indexing_task: asyncio.Task | None = None
 
-    async def start_background_indexing(self) -> None:
+    async def start_background_indexing(
+        self, operation: ProgressTracker | None = None
+    ) -> None:
         """Start background indexing of configured sources."""
+        operation = operation or create_noop_operation()
         if (
             not self.app_context.auto_indexing
             or len(self.app_context.auto_indexing.sources) == 0
@@ -48,15 +53,22 @@ class AutoIndexingService:
 
         auto_sources = [source.uri for source in self.app_context.auto_indexing.sources]
         self.log.info("Starting background indexing", num_sources=len(auto_sources))
-        self._indexing_task = asyncio.create_task(self._index_sources(auto_sources))
+        self._indexing_task = asyncio.create_task(
+            self._index_sources(auto_sources, operation)
+        )
 
-    async def _index_sources(self, sources: list[str]) -> None:
+    async def _index_sources(
+        self, sources: list[str], operation: ProgressTracker | None = None
+    ) -> None:
         """Index all configured sources in the background."""
+        operation = operation or create_noop_operation()
         async with self.session_factory() as session:
-            queue_service = QueueService(session=session)
+            queue_service = QueueService(session_factory=self.session_factory)
             service = create_code_indexing_application_service(
                 app_context=self.app_context,
                 session=session,
+                session_factory=self.session_factory,
+                operation=operation,
             )
 
             for source in sources:

{kodit-0.4.1 → kodit-0.4.2}/src/kodit/application/services/code_indexing_application_service.py

@@ -6,8 +6,11 @@ from datetime import UTC, datetime
 import structlog
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from kodit.application.services.reporting import (
+    OperationType,
+    ProgressTracker,
+)
 from kodit.domain.entities import Index, Snippet
-from kodit.domain.interfaces import ProgressCallback
 from kodit.domain.protocols import IndexRepository
 from kodit.domain.services.bm25_service import BM25DomainService
 from kodit.domain.services.embedding_service import EmbeddingDomainService
@@ -25,7 +28,6 @@ from kodit.domain.value_objects import (
     SnippetSearchFilters,
 )
 from kodit.log import log_event
-from kodit.reporting import Reporter
 
 
 class CodeIndexingApplicationService:
@@ -41,6 +43,7 @@ class CodeIndexingApplicationService:
         text_search_service: EmbeddingDomainService,
         enrichment_service: EnrichmentDomainService,
         session: AsyncSession,
+        operation: ProgressTracker,
     ) -> None:
         """Initialize the code indexing application service."""
         self.index_domain_service = indexing_domain_service
@@ -51,6 +54,7 @@ class CodeIndexingApplicationService:
         self.text_search_service = text_search_service
         self.enrichment_service = enrichment_service
         self.session = session
+        self.operation = operation
         self.log = structlog.get_logger(__name__)
 
     async def does_index_exist(self, uri: str) -> bool:
@@ -60,107 +64,117 @@ class CodeIndexingApplicationService:
         existing_index = await self.index_repository.get_by_uri(sanitized_uri)
         return existing_index is not None
 
-    async def create_index_from_uri(
-        self, uri: str, progress_callback: ProgressCallback | None = None
-    ) -> Index:
+    async def create_index_from_uri(self, uri: str) -> Index:
         """Create a new index for a source."""
         log_event("kodit.index.create")
+        with self.operation.create_child(OperationType.CREATE_INDEX.value) as operation:
+            # Check if index already exists
+            sanitized_uri, _ = self.index_domain_service.sanitize_uri(uri)
+            self.log.info("Creating index from URI", uri=str(sanitized_uri))
+            existing_index = await self.index_repository.get_by_uri(sanitized_uri)
+            if existing_index:
+                self.log.debug(
+                    "Index already exists",
+                    uri=str(sanitized_uri),
+                    index_id=existing_index.id,
+                )
+                return existing_index
 
-        # Check if index already exists
-        sanitized_uri, _ = self.index_domain_service.sanitize_uri(uri)
-        existing_index = await self.index_repository.get_by_uri(sanitized_uri)
-        if existing_index:
-            self.log.debug(
-                "Index already exists",
-                uri=str(sanitized_uri),
-                index_id=existing_index.id,
-            )
-            return existing_index
-
-        # Only prepare working copy if we need to create a new index
-        working_copy = await self.index_domain_service.prepare_index(
-            uri, progress_callback
-        )
+            # Only prepare working copy if we need to create a new index
+            self.log.info("Preparing working copy", uri=str(sanitized_uri))
+            working_copy = await self.index_domain_service.prepare_index(uri, operation)
 
-        # Create new index
-        index = await self.index_repository.create(sanitized_uri, working_copy)
-        await self.session.commit()
-        return index
+            # Create new index
+            self.log.info("Creating index", uri=str(sanitized_uri))
+            index = await self.index_repository.create(sanitized_uri, working_copy)
+            await self.session.commit()
+            return index
 
-    async def run_index(
-        self, index: Index, progress_callback: ProgressCallback | None = None
-    ) -> None:
+    async def run_index(self, index: Index) -> None:
         """Run the complete indexing process for a specific index."""
-        log_event("kodit.index.run")
-
-        if not index or not index.id:
-            msg = f"Index has no ID: {index}"
-            raise ValueError(msg)
-
-        # Refresh working copy
-        index.source.working_copy = (
-            await self.index_domain_service.refresh_working_copy(
-                index.source.working_copy
-            )
-        )
-        if len(index.source.working_copy.changed_files()) == 0:
-            self.log.info("No new changes to index", index_id=index.id)
-            return
-
-        # Delete the old snippets from the files that have changed
-        await self.index_repository.delete_snippets_by_file_ids(
-            [file.id for file in index.source.working_copy.changed_files() if file.id]
-        )
-
-        # Extract and create snippets (domain service handles progress)
-        self.log.info("Creating snippets for files", index_id=index.id)
-        index = await self.index_domain_service.extract_snippets_from_index(
-            index=index, progress_callback=progress_callback
-        )
-
-        await self.index_repository.update(index)
-        await self.session.flush()
-
-        # Refresh index to get snippets with IDs, required as a ref for subsequent steps
-        flushed_index = await self.index_repository.get(index.id)
-        if not flushed_index:
-            msg = f"Index {index.id} not found after snippet extraction"
-            raise ValueError(msg)
-        index = flushed_index
-        if len(index.snippets) == 0:
-            self.log.info("No snippets to index after extraction", index_id=index.id)
-            return
-
-        # Create BM25 index
-        self.log.info("Creating keyword index")
-        await self._create_bm25_index(index.snippets, progress_callback)
-
-        # Create code embeddings
-        self.log.info("Creating semantic code index")
-        await self._create_code_embeddings(index.snippets, progress_callback)
-
-        # Enrich snippets
-        self.log.info("Enriching snippets", num_snippets=len(index.snippets))
-        enriched_snippets = await self.index_domain_service.enrich_snippets_in_index(
-            snippets=index.snippets, progress_callback=progress_callback
-        )
-        # Update snippets in repository
-        await self.index_repository.update_snippets(index.id, enriched_snippets)
+        # Create a new operation
+        with self.operation.create_child(OperationType.RUN_INDEX.value) as operation:
+            # TODO(philwinder): Move this into a reporter # noqa: TD003, FIX002
+            log_event("kodit.index.run")
+
+            if not index or not index.id:
+                msg = f"Index has no ID: {index}"
+                raise ValueError(msg)
+
+            # Refresh working copy
+            with operation.create_child("Refresh working copy") as step:
+                index.source.working_copy = (
+                    await self.index_domain_service.refresh_working_copy(
+                        index.source.working_copy, step
+                    )
+                )
+                if len(index.source.working_copy.changed_files()) == 0:
+                    self.log.info("No new changes to index", index_id=index.id)
+                    step.skip("No new changes to index")
+                    return
+
+            # Delete the old snippets from the files that have changed
+            with operation.create_child("Delete old snippets") as step:
+                await self.index_repository.delete_snippets_by_file_ids(
+                    [
+                        file.id
+                        for file in index.source.working_copy.changed_files()
+                        if file.id
+                    ]
+                )
 
-        # Create text embeddings (on enriched content)
-        self.log.info("Creating semantic text index")
-        await self._create_text_embeddings(enriched_snippets, progress_callback)
+            # Extract and create snippets (domain service handles progress)
+            with operation.create_child("Extract snippets") as step:
+                index = await self.index_domain_service.extract_snippets_from_index(
+                    index=index, step=step
+                )
+                await self.index_repository.update(index)
+
+                # Refresh index to get snippets with IDs, required for subsequent steps
+                flushed_index = await self.index_repository.get(index.id)
+                if not flushed_index:
+                    msg = f"Index {index.id} not found after snippet extraction"
+                    raise ValueError(msg)
+                index = flushed_index
+                if len(index.snippets) == 0:
+                    self.log.info(
+                        "No snippets to index after extraction", index_id=index.id
+                    )
+                    step.skip("No snippets to index after extraction")
+                    return
+
+            # Create BM25 index
+            self.log.info("Creating keyword index")
+            with operation.create_child("Create BM25 index") as step:
+                await self._create_bm25_index(index.snippets)
+
+            # Create code embeddings
+            with operation.create_child("Create code embeddings") as step:
+                await self._create_code_embeddings(index.snippets, step)
+
+            # Enrich snippets
+            with operation.create_child("Enrich snippets") as step:
+                enriched_snippets = (
+                    await self.index_domain_service.enrich_snippets_in_index(
+                        snippets=index.snippets,
+                        reporting_step=step,
+                    )
+                )
+                # Update snippets in repository
+                await self.index_repository.update_snippets(index.id, enriched_snippets)
 
-        # Update index timestamp
-        await self.index_repository.update_index_timestamp(index.id)
+            # Create text embeddings (on enriched content)
+            with operation.create_child("Create text embeddings") as step:
+                await self._create_text_embeddings(enriched_snippets, step)
 
-        # Now that all file dependencies have been captured, enact the file processing
-        # statuses
-        index.source.working_copy.clear_file_processing_statuses()
-        await self.index_repository.update(index)
+            # Update index timestamp
+            with operation.create_child("Update index timestamp") as step:
+                await self.index_repository.update_index_timestamp(index.id)
 
-        # Single transaction commit for the entire operation
-        await self.session.commit()
+            # After indexing, clear the file processing statuses
+            with operation.create_child("Clear file processing statuses") as step:
+                index.source.working_copy.clear_file_processing_statuses()
+                await self.index_repository.update(index)
 
     async def search(self, request: MultiSearchRequest) -> list[MultiSearchResult]:
         """Search for relevant snippets across all indexes."""
@@ -312,15 +326,7 @@ class CodeIndexingApplicationService:
         ]
 
     # FUTURE: BM25 index enriched content too
-    async def _create_bm25_index(
-        self, snippets: list[Snippet], progress_callback: ProgressCallback | None = None
-    ) -> None:
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start("bm25_index", len(snippets), "Creating keyword index...")
-
-        for _snippet in snippets:
-            pass
-
+    async def _create_bm25_index(self, snippets: list[Snippet]) -> None:
         await self.bm25_service.index_documents(
             IndexRequest(
                 documents=[
@@ -331,16 +337,10 @@ class CodeIndexingApplicationService:
             )
         )
 
-        await reporter.done("bm25_index", "Keyword index created")
-
     async def _create_code_embeddings(
-        self, snippets: list[Snippet], progress_callback: ProgressCallback | None = None
+        self, snippets: list[Snippet], reporting_step: ProgressTracker
     ) -> None:
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start(
-            "code_embeddings", len(snippets), "Creating code embeddings..."
-        )
-
+        reporting_step.set_total(len(snippets))
         processed = 0
         async for result in self.code_search_service.index_documents(
             IndexRequest(
@@ -352,23 +352,11 @@ class CodeIndexingApplicationService:
             )
         ):
             processed += len(result)
-            await reporter.step(
-                "code_embeddings",
-                processed,
-                len(snippets),
-                "Creating code embeddings...",
-            )
-
-        await reporter.done("code_embeddings")
+            reporting_step.set_current(processed)
 
     async def _create_text_embeddings(
-        self, snippets: list[Snippet], progress_callback: ProgressCallback | None = None
+        self, snippets: list[Snippet], reporting_step: ProgressTracker
     ) -> None:
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start(
-            "text_embeddings", len(snippets), "Creating text embeddings..."
-        )
-
         # Only create text embeddings for snippets that have summary content
         documents_with_summaries = []
         for snippet in snippets:
@@ -384,22 +372,16 @@ class CodeIndexingApplicationService:
                     continue
 
         if not documents_with_summaries:
-            await reporter.done("text_embeddings", "No summaries to index")
+            reporting_step.skip("No snippets with summaries to create text embeddings")
             return
 
+        reporting_step.set_total(len(documents_with_summaries))
         processed = 0
         async for result in self.text_search_service.index_documents(
             IndexRequest(documents=documents_with_summaries)
         ):
             processed += len(result)
-            await reporter.step(
-                "text_embeddings",
-                processed,
-                len(snippets),
-                "Creating text embeddings...",
-            )
-
-        await reporter.done("text_embeddings")
+            reporting_step.set_current(processed)
 
     async def delete_index(self, index: Index) -> None:
         """Delete an index."""