kodit 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

kodit/domain/services/index_service.py

@@ -8,7 +8,8 @@ import structlog
 from pydantic import AnyUrl
 
 import kodit.domain.entities as domain_entities
-from kodit.domain.interfaces import ProgressCallback
+from kodit.application.factories.reporting_factory import create_noop_operation
+from kodit.application.services.reporting import ProgressTracker
 from kodit.domain.services.enrichment_service import EnrichmentDomainService
 from kodit.domain.value_objects import (
     EnrichmentIndexRequest,
@@ -21,7 +22,6 @@ from kodit.infrastructure.cloning.metadata import FileMetadataExtractor
 from kodit.infrastructure.git.git_utils import is_valid_clone_target
 from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
 from kodit.infrastructure.slicing.slicer import Slicer
-from kodit.reporting import Reporter
 from kodit.utils.path_utils import path_from_uri
 
 
@@ -58,27 +58,23 @@ class IndexDomainService:
     async def prepare_index(
         self,
         uri_or_path_like: str,  # Must include user/pass, etc
-        progress_callback: ProgressCallback | None = None,
+        step: ProgressTracker | None = None,
     ) -> domain_entities.WorkingCopy:
         """Prepare an index by scanning files and creating working copy."""
+        step = step or create_noop_operation()
+        self.log.info("Preparing index")
         sanitized_uri, source_type = self.sanitize_uri(uri_or_path_like)
-        reporter = Reporter(self.log, progress_callback)
         self.log.info("Preparing source", uri=str(sanitized_uri))
 
         if source_type == domain_entities.SourceType.FOLDER:
-            await reporter.start("prepare_index", 1, "Scanning source...")
             local_path = path_from_uri(str(sanitized_uri))
         elif source_type == domain_entities.SourceType.GIT:
             source_type = domain_entities.SourceType.GIT
             git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
-            await reporter.start("prepare_index", 1, "Cloning source...")
-            local_path = await git_working_copy_provider.prepare(uri_or_path_like)
-            await reporter.done("prepare_index")
+            local_path = await git_working_copy_provider.prepare(uri_or_path_like, step)
         else:
             raise ValueError(f"Unsupported source: {uri_or_path_like}")
 
-        await reporter.done("prepare_index")
-
         return domain_entities.WorkingCopy(
             remote_uri=sanitized_uri,
             cloned_path=local_path,
@@ -89,9 +85,10 @@ class IndexDomainService:
     async def extract_snippets_from_index(
         self,
         index: domain_entities.Index,
-        progress_callback: ProgressCallback | None = None,
+        step: ProgressTracker | None = None,
     ) -> domain_entities.Index:
         """Extract code snippets from files in the index."""
+        step = step or create_noop_operation()
         file_count = len(index.source.working_copy.files)
 
         self.log.info(
@@ -127,40 +124,28 @@ class IndexDomainService:
             languages=lang_files_map.keys(),
         )
 
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start(
-            "extract_snippets",
-            len(lang_files_map.keys()),
-            "Extracting code snippets...",
-        )
-
         # Calculate snippets for each language
         slicer = Slicer()
+        await step.set_total(len(lang_files_map.keys()))
         for i, (lang, lang_files) in enumerate(lang_files_map.items()):
-            await reporter.step(
-                "extract_snippets",
-                i,
-                len(lang_files_map.keys()),
-                f"Extracting code snippets for {lang}...",
-            )
+            await step.set_current(i, f"Extracting snippets for {lang}")
             s = slicer.extract_snippets(lang_files, language=lang)
             index.snippets.extend(s)
 
-        await reporter.done("extract_snippets")
         return index
 
     async def enrich_snippets_in_index(
         self,
         snippets: list[domain_entities.Snippet],
-        progress_callback: ProgressCallback | None = None,
+        reporting_step: ProgressTracker | None = None,
     ) -> list[domain_entities.Snippet]:
         """Enrich snippets with AI-generated summaries."""
+        reporting_step = reporting_step or create_noop_operation()
         if not snippets or len(snippets) == 0:
+            await reporting_step.skip("No snippets to enrich")
             return snippets
 
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start("enrichment", len(snippets), "Enriching snippets...")
-
+        await reporting_step.set_total(len(snippets))
         snippet_map = {snippet.id: snippet for snippet in snippets if snippet.id}
 
         enrichment_request = EnrichmentIndexRequest(
@@ -177,11 +162,10 @@ class IndexDomainService:
             snippet_map[result.snippet_id].add_summary(result.text)
 
             processed += 1
-            await reporter.step(
-                "enrichment", processed, len(snippets), "Enriching snippets..."
+            await reporting_step.set_current(
+                processed, f"Enriching snippets for {processed} snippets"
             )
 
-        await reporter.done("enrichment")
         return list(snippet_map.values())
 
     def sanitize_uri(
@@ -207,15 +191,14 @@ class IndexDomainService:
     async def refresh_working_copy(
         self,
         working_copy: domain_entities.WorkingCopy,
-        progress_callback: ProgressCallback | None = None,
+        step: ProgressTracker | None = None,
     ) -> domain_entities.WorkingCopy:
         """Refresh the working copy."""
+        step = step or create_noop_operation()
         metadata_extractor = FileMetadataExtractor(working_copy.source_type)
-        reporter = Reporter(self.log, progress_callback)
-
         if working_copy.source_type == domain_entities.SourceType.GIT:
             git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
-            await git_working_copy_provider.sync(str(working_copy.remote_uri))
+            await git_working_copy_provider.sync(str(working_copy.remote_uri), step)
 
         current_file_paths = working_copy.list_filesystem_paths(
             GitIgnorePatternProvider(working_copy.cloned_path)
@@ -241,19 +224,12 @@ class IndexDomainService:
 
         # Setup reporter
         processed = 0
-        await reporter.start(
-            "refresh_working_copy", num_files_to_process, "Refreshing working copy..."
-        )
+        await step.set_total(num_files_to_process)
 
         # First check to see if any files have been deleted
         for file_path in deleted_file_paths:
             processed += 1
-            await reporter.step(
-                "refresh_working_copy",
-                processed,
-                num_files_to_process,
-                f"Deleted {file_path.name}",
-            )
+            await step.set_current(processed, f"Deleting file {file_path}")
             previous_files_map[
                 file_path
             ].file_processing_status = domain_entities.FileProcessingStatus.DELETED
@@ -261,12 +237,7 @@ class IndexDomainService:
         # Then check to see if there are any new files
         for file_path in new_file_paths:
             processed += 1
-            await reporter.step(
-                "refresh_working_copy",
-                processed,
-                num_files_to_process,
-                f"New {file_path.name}",
-            )
+            await step.set_current(processed, f"Adding new file {file_path}")
             try:
                 working_copy.files.append(
                     await metadata_extractor.extract(file_path=file_path)
@@ -278,12 +249,7 @@ class IndexDomainService:
         # Finally check if there are any modified files
         for file_path in modified_file_paths:
             processed += 1
-            await reporter.step(
-                "refresh_working_copy",
-                processed,
-                num_files_to_process,
-                f"Modified {file_path.name}",
-            )
+            await step.set_current(processed, f"Modifying file {file_path}")
             try:
                 previous_file = previous_files_map[file_path]
                 new_file = await metadata_extractor.extract(file_path=file_path)

kodit/domain/services/task_status_query_service.py

@@ -0,0 +1,19 @@
+"""Domain service for querying task status."""
+
+from kodit.domain.entities import TaskStatus
+from kodit.domain.protocols import TaskStatusRepository
+from kodit.domain.value_objects import TrackableType
+
+
+class TaskStatusQueryService:
+    """Query service for task status information."""
+
+    def __init__(self, repository: TaskStatusRepository) -> None:
+        """Initialize the task status query service."""
+        self._repository = repository
+
+    async def get_index_status(self, index_id: int) -> list[TaskStatus]:
+        """Get the status of tasks for a specific index."""
+        return await self._repository.load_with_hierarchy(
+            trackable_type=TrackableType.INDEX.value, trackable_id=index_id
+        )

kodit/domain/value_objects.py

@@ -3,7 +3,7 @@
 import json
 from dataclasses import dataclass
 from datetime import datetime
-from enum import Enum, IntEnum
+from enum import Enum, IntEnum, StrEnum
 from pathlib import Path
 from typing import ClassVar
 
@@ -390,18 +390,18 @@ class IndexRunRequest:
 
 
 @dataclass
-class ProgressEvent:
-    """Domain model for progress events."""
+class ProgressState:
+    """Progress state."""
 
-    operation: str
-    current: int
-    total: int
-    message: str | None = None
+    current: int = 0
+    total: int = 0
+    operation: str = ""
+    message: str = ""
 
     @property
     def percentage(self) -> float:
         """Calculate the percentage of completion."""
-        return (self.current / self.total * 100) if self.total > 0 else 0.0
+        return (self.current / self.total) * 100 if self.total > 0 else 0.0
 
 
 @dataclass
@@ -662,3 +662,48 @@ class QueuePriority(IntEnum):
 
     BACKGROUND = 10
     USER_INITIATED = 50
+
+
+# Reporting value objects
+
+
+class ReportingState(StrEnum):
+    """Reporting state."""
+
+    STARTED = "started"
+    IN_PROGRESS = "in_progress"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+    @staticmethod
+    def is_terminal(state: "ReportingState") -> bool:
+        """Check if a state is completed."""
+        return state in [
+            ReportingState.COMPLETED,
+            ReportingState.FAILED,
+            ReportingState.SKIPPED,
+        ]
+
+
+class TrackableType(StrEnum):
+    """Trackable type."""
+
+    INDEX = "indexes"
+
+
+class TaskOperation(StrEnum):
+    """Task operation."""
+
+    ROOT = "kodit.root"
+    CREATE_INDEX = "kodit.index.create"
+    RUN_INDEX = "kodit.index.run"
+    REFRESH_WORKING_COPY = "kodit.index.run.refresh_working_copy"
+    DELETE_OLD_SNIPPETS = "kodit.index.run.delete_old_snippets"
+    EXTRACT_SNIPPETS = "kodit.index.run.extract_snippets"
+    CREATE_BM25_INDEX = "kodit.index.run.create_bm25_index"
+    CREATE_CODE_EMBEDDINGS = "kodit.index.run.create_code_embeddings"
+    ENRICH_SNIPPETS = "kodit.index.run.enrich_snippets"
+    CREATE_TEXT_EMBEDDINGS = "kodit.index.run.create_text_embeddings"
+    UPDATE_INDEX_TIMESTAMP = "kodit.index.run.update_index_timestamp"
+    CLEAR_FILE_PROCESSING_STATUSES = "kodit.index.run.clear_file_processing_statuses"

kodit/infrastructure/api/v1/dependencies.py

@@ -1,13 +1,13 @@
 """FastAPI dependencies for the REST API."""
 
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Callable
 from typing import Annotated, cast
 
 from fastapi import Depends, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.application.factories.code_indexing_factory import (
-    create_code_indexing_application_service,
+    create_server_code_indexing_application_service,
 )
 from kodit.application.services.code_indexing_application_service import (
     CodeIndexingApplicationService,
@@ -15,8 +15,12 @@ from kodit.application.services.code_indexing_application_service import (
 from kodit.application.services.queue_service import QueueService
 from kodit.config import AppContext
 from kodit.domain.services.index_query_service import IndexQueryService
+from kodit.domain.services.task_status_query_service import TaskStatusQueryService
 from kodit.infrastructure.indexing.fusion_service import ReciprocalRankFusionService
-from kodit.infrastructure.sqlalchemy.index_repository import SqlAlchemyIndexRepository
+from kodit.infrastructure.sqlalchemy.index_repository import create_index_repository
+from kodit.infrastructure.sqlalchemy.task_status_repository import (
+    create_task_status_repository,
+)
 
 
 def get_app_context(request: Request) -> AppContext:
@@ -42,12 +46,25 @@ async def get_db_session(
 DBSessionDep = Annotated[AsyncSession, Depends(get_db_session)]
 
 
+async def get_db_session_factory(
+    app_context: AppContextDep,
+) -> AsyncGenerator[Callable[[], AsyncSession], None]:
+    """Get database session dependency."""
+    db = await app_context.get_db()
+    yield db.session_factory
+
+
+DBSessionFactoryDep = Annotated[
+    Callable[[], AsyncSession], Depends(get_db_session_factory)
+]
+
+
 async def get_index_query_service(
-    session: DBSessionDep,
+    session_factory: DBSessionFactoryDep,
 ) -> IndexQueryService:
     """Get index query service dependency."""
     return IndexQueryService(
-        index_repository=SqlAlchemyIndexRepository(session=session),
+        index_repository=create_index_repository(session_factory=session_factory),
         fusion_service=ReciprocalRankFusionService(),
     )
 
@@ -57,13 +74,10 @@ IndexQueryServiceDep = Annotated[IndexQueryService, Depends(get_index_query_serv
 
 async def get_indexing_app_service(
     app_context: AppContextDep,
-    session: DBSessionDep,
+    session_factory: DBSessionFactoryDep,
 ) -> CodeIndexingApplicationService:
     """Get indexing application service dependency."""
-    return create_code_indexing_application_service(
-        app_context=app_context,
-        session=session,
-    )
+    return create_server_code_indexing_application_service(app_context, session_factory)
 
 
 IndexingAppServiceDep = Annotated[
@@ -72,12 +86,26 @@ IndexingAppServiceDep = Annotated[
 
 
 async def get_queue_service(
-    session: DBSessionDep,
+    session_factory: DBSessionFactoryDep,
 ) -> QueueService:
     """Get queue service dependency."""
     return QueueService(
-        session=session,
+        session_factory=session_factory,
     )
 
 
 QueueServiceDep = Annotated[QueueService, Depends(get_queue_service)]
+
+
+async def get_task_status_query_service(
+    session_factory: DBSessionFactoryDep,
+) -> TaskStatusQueryService:
+    """Get task status query service dependency."""
+    return TaskStatusQueryService(
+        repository=create_task_status_repository(session_factory=session_factory)
+    )
+
+
+TaskStatusQueryServiceDep = Annotated[
+    TaskStatusQueryService, Depends(get_task_status_query_service)
+]

kodit/infrastructure/api/v1/routers/indexes.py

@@ -9,6 +9,7 @@ from kodit.infrastructure.api.v1.dependencies import (
     IndexingAppServiceDep,
     IndexQueryServiceDep,
     QueueServiceDep,
+    TaskStatusQueryServiceDep,
 )
 from kodit.infrastructure.api.v1.schemas.index import (
     IndexAttributes,
@@ -18,6 +19,11 @@ from kodit.infrastructure.api.v1.schemas.index import (
     IndexListResponse,
     IndexResponse,
 )
+from kodit.infrastructure.api.v1.schemas.task_status import (
+    TaskStatusAttributes,
+    TaskStatusData,
+    TaskStatusListResponse,
+)
 
 router = APIRouter(
     prefix="/api/v1/indexes",
@@ -103,6 +109,45 @@ async def get_index(
     )
 
 
+@router.get(
+    "/{index_id}/status",
+    responses={404: {"description": "Index not found"}},
+)
+async def get_index_status(
+    index_id: int,
+    query_service: IndexQueryServiceDep,
+    status_service: TaskStatusQueryServiceDep,
+) -> TaskStatusListResponse:
+    """Get the status of tasks for an index."""
+    # Verify the index exists
+    index = await query_service.get_index_by_id(index_id)
+    if not index:
+        raise HTTPException(status_code=404, detail="Index not found")
+
+    # Get all task statuses for this index
+    progress_trackers = await status_service.get_index_status(index_id)
+
+    # Convert progress trackers to API response format
+    task_statuses = []
+    for _i, status in enumerate(progress_trackers):
+        task_statuses.append(
+            TaskStatusData(
+                id=status.id,
+                attributes=TaskStatusAttributes(
+                    step=status.operation,
+                    state=status.state,
+                    progress=status.completion_percent,
+                    total=status.total,
+                    current=status.current,
+                    created_at=status.created_at,
+                    updated_at=status.updated_at,
+                ),
+            )
+        )
+
+    return TaskStatusListResponse(data=task_statuses)
+
+
 @router.delete(
     "/{index_id}", status_code=204, responses={404: {"description": "Index not found"}}
 )

kodit/infrastructure/api/v1/schemas/task_status.py

@@ -0,0 +1,39 @@
+"""JSON:API schemas for task status operations."""
+
+from datetime import datetime
+
+from pydantic import BaseModel, Field
+
+
+class TaskStatusAttributes(BaseModel):
+    """Task status attributes for JSON:API responses."""
+
+    step: str = Field(..., description="Name of the task/operation")
+    state: str = Field(..., description="Current state of the task")
+    progress: float = Field(
+        default=0.0, ge=0.0, le=100.0, description="Progress percentage (0-100)"
+    )
+    total: int = Field(default=0, description="Total number of items to process")
+    current: int = Field(default=0, description="Current number of items processed")
+    created_at: datetime | None = Field(default=None, description="Task start time")
+    updated_at: datetime | None = Field(default=None, description="Last update time")
+
+
+class TaskStatusData(BaseModel):
+    """Task status data for JSON:API responses."""
+
+    type: str = "task_status"
+    id: str
+    attributes: TaskStatusAttributes
+
+
+class TaskStatusResponse(BaseModel):
+    """JSON:API response for single task status."""
+
+    data: TaskStatusData
+
+
+class TaskStatusListResponse(BaseModel):
+    """JSON:API response for task status list."""
+
+    data: list[TaskStatusData]

kodit/infrastructure/cloning/git/working_copy.py

@@ -1,5 +1,6 @@
 """Working copy provider for git-based sources."""
 
+import asyncio
 import hashlib
 import shutil
 from pathlib import Path
@@ -7,6 +8,8 @@ from pathlib import Path
 import git
 import structlog
 
+from kodit.application.factories.reporting_factory import create_noop_operation
+from kodit.application.services.reporting import ProgressTracker
 from kodit.domain.entities import WorkingCopy
 
 
@@ -25,18 +28,48 @@ class GitWorkingCopyProvider:
         dir_name = f"repo-{dir_hash}"
         return self.clone_dir / dir_name
 
-    async def prepare(self, uri: str) -> Path:
+    async def prepare(
+        self,
+        uri: str,
+        step: ProgressTracker | None = None,
+    ) -> Path:
         """Prepare a Git working copy."""
+        step = step or create_noop_operation()
         sanitized_uri = WorkingCopy.sanitize_git_url(uri)
         clone_path = self.get_clone_path(uri)
         clone_path.mkdir(parents=True, exist_ok=True)
 
+        step_record = []
+        await step.set_total(12)
+
+        def _clone_progress_callback(
+            a: int, _: str | float | None, __: str | float | None, _d: str
+        ) -> None:
+            if a not in step_record:
+                step_record.append(a)
+
+            # Git reports a really weird format. This is a quick hack to get some
+            # progress.
+            # Normally this would fail because the loop is already running,
+            # but in this case, this callback is called by some git sub-thread.
+            asyncio.run(
+                step.set_current(
+                    len(step_record), f"Cloning repository ({step_record[-1]})"
+                )
+            )
+
         try:
             self.log.info(
                 "Cloning repository", uri=sanitized_uri, clone_path=str(clone_path)
             )
             # Use the original URI for cloning (with credentials if present)
-            git.Repo.clone_from(uri, clone_path)
+            options = ["--depth=1", "--single-branch"]
+            git.Repo.clone_from(
+                uri,
+                clone_path,
+                progress=_clone_progress_callback,
+                multi_options=options,
+            )
         except git.GitCommandError as e:
             if "already exists and is not an empty directory" not in str(e):
                 msg = f"Failed to clone repository: {e}"
@@ -45,8 +78,9 @@ class GitWorkingCopyProvider:
 
         return clone_path
 
-    async def sync(self, uri: str) -> Path:
+    async def sync(self, uri: str, step: ProgressTracker | None = None) -> Path:
         """Refresh a Git working copy."""
+        step = step or create_noop_operation()
         clone_path = self.get_clone_path(uri)
 
         # Check if the clone directory exists and is a valid Git repository
@@ -54,9 +88,10 @@ class GitWorkingCopyProvider:
             self.log.info(
                 "Clone directory does not exist or is not a Git repository, "
                 "preparing...",
-                uri=uri, clone_path=str(clone_path)
+                uri=uri,
+                clone_path=str(clone_path),
             )
-            return await self.prepare(uri)
+            return await self.prepare(uri, step)
 
         try:
             repo = git.Repo(clone_path)
@@ -64,10 +99,11 @@ class GitWorkingCopyProvider:
         except git.InvalidGitRepositoryError:
             self.log.warning(
                 "Invalid Git repository found, re-cloning...",
-                uri=uri, clone_path=str(clone_path)
+                uri=uri,
+                clone_path=str(clone_path),
             )
             # Remove the invalid directory and re-clone
             shutil.rmtree(clone_path)
-            return await self.prepare(uri)
+            return await self.prepare(uri, step)
 
         return clone_path

kodit/infrastructure/embedding/embedding_factory.py

@@ -1,5 +1,7 @@
 """Factory for creating embedding services with DDD architecture."""
 
+from collections.abc import Callable
+
 import structlog
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -24,7 +26,7 @@ from kodit.infrastructure.embedding.vectorchord_vector_search_repository import
     VectorChordVectorSearchRepository,
 )
 from kodit.infrastructure.sqlalchemy.embedding_repository import (
-    SqlAlchemyEmbeddingRepository,
+    create_embedding_repository,
 )
 from kodit.infrastructure.sqlalchemy.entities import EmbeddingType
 from kodit.log import log_event
@@ -36,12 +38,15 @@ def _get_endpoint_configuration(app_context: AppContext) -> Endpoint | None:
 
 
 def embedding_domain_service_factory(
-    task_name: TaskName, app_context: AppContext, session: AsyncSession
+    task_name: TaskName,
+    app_context: AppContext,
+    session: AsyncSession,
+    session_factory: Callable[[], AsyncSession],
 ) -> EmbeddingDomainService:
     """Create an embedding domain service."""
     structlog.get_logger(__name__)
     # Create embedding repository
-    embedding_repository = SqlAlchemyEmbeddingRepository(session=session)
+    embedding_repository = create_embedding_repository(session_factory=session_factory)
 
     # Create embedding provider
     embedding_provider: EmbeddingProvider | None = None