kodit 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

kodit/domain/services/index_service.py

@@ -8,7 +8,8 @@ import structlog
 from pydantic import AnyUrl
 
 import kodit.domain.entities as domain_entities
-from kodit.domain.interfaces import ProgressCallback
+from kodit.application.factories.reporting_factory import create_noop_operation
+from kodit.application.services.reporting import ProgressTracker
 from kodit.domain.services.enrichment_service import EnrichmentDomainService
 from kodit.domain.value_objects import (
     EnrichmentIndexRequest,
@@ -21,7 +22,6 @@ from kodit.infrastructure.cloning.metadata import FileMetadataExtractor
 from kodit.infrastructure.git.git_utils import is_valid_clone_target
 from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
 from kodit.infrastructure.slicing.slicer import Slicer
-from kodit.reporting import Reporter
 from kodit.utils.path_utils import path_from_uri
 
 
@@ -58,27 +58,23 @@ class IndexDomainService:
     async def prepare_index(
         self,
         uri_or_path_like: str,  # Must include user/pass, etc
-        progress_callback: ProgressCallback | None = None,
+        step: ProgressTracker | None = None,
     ) -> domain_entities.WorkingCopy:
         """Prepare an index by scanning files and creating working copy."""
+        step = step or create_noop_operation()
+        self.log.info("Preparing index")
         sanitized_uri, source_type = self.sanitize_uri(uri_or_path_like)
-        reporter = Reporter(self.log, progress_callback)
         self.log.info("Preparing source", uri=str(sanitized_uri))
 
         if source_type == domain_entities.SourceType.FOLDER:
-            await reporter.start("prepare_index", 1, "Scanning source...")
             local_path = path_from_uri(str(sanitized_uri))
         elif source_type == domain_entities.SourceType.GIT:
             source_type = domain_entities.SourceType.GIT
             git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
-            await reporter.start("prepare_index", 1, "Cloning source...")
-            local_path = await git_working_copy_provider.prepare(uri_or_path_like)
-            await reporter.done("prepare_index")
+            local_path = await git_working_copy_provider.prepare(uri_or_path_like, step)
         else:
             raise ValueError(f"Unsupported source: {uri_or_path_like}")
 
-        await reporter.done("prepare_index")
-
         return domain_entities.WorkingCopy(
             remote_uri=sanitized_uri,
             cloned_path=local_path,
@@ -89,9 +85,10 @@ class IndexDomainService:
     async def extract_snippets_from_index(
         self,
         index: domain_entities.Index,
-        progress_callback: ProgressCallback | None = None,
+        step: ProgressTracker | None = None,
     ) -> domain_entities.Index:
         """Extract code snippets from files in the index."""
+        step = step or create_noop_operation()
         file_count = len(index.source.working_copy.files)
 
         self.log.info(
@@ -127,40 +124,28 @@ class IndexDomainService:
             languages=lang_files_map.keys(),
         )
 
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start(
-            "extract_snippets",
-            len(lang_files_map.keys()),
-            "Extracting code snippets...",
-        )
-
         # Calculate snippets for each language
         slicer = Slicer()
+        step.set_total(len(lang_files_map.keys()))
         for i, (lang, lang_files) in enumerate(lang_files_map.items()):
-            await reporter.step(
-                "extract_snippets",
-                i,
-                len(lang_files_map.keys()),
-                f"Extracting code snippets for {lang}...",
-            )
+            step.set_current(i)
             s = slicer.extract_snippets(lang_files, language=lang)
             index.snippets.extend(s)
 
-        await reporter.done("extract_snippets")
         return index
 
     async def enrich_snippets_in_index(
         self,
         snippets: list[domain_entities.Snippet],
-        progress_callback: ProgressCallback | None = None,
+        reporting_step: ProgressTracker | None = None,
     ) -> list[domain_entities.Snippet]:
         """Enrich snippets with AI-generated summaries."""
+        reporting_step = reporting_step or create_noop_operation()
         if not snippets or len(snippets) == 0:
+            reporting_step.skip("No snippets to enrich")
             return snippets
 
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start("enrichment", len(snippets), "Enriching snippets...")
-
+        reporting_step.set_total(len(snippets))
         snippet_map = {snippet.id: snippet for snippet in snippets if snippet.id}
 
         enrichment_request = EnrichmentIndexRequest(
@@ -177,11 +162,8 @@ class IndexDomainService:
             snippet_map[result.snippet_id].add_summary(result.text)
 
             processed += 1
-            await reporter.step(
-                "enrichment", processed, len(snippets), "Enriching snippets..."
-            )
+            reporting_step.set_current(processed)
 
-        await reporter.done("enrichment")
         return list(snippet_map.values())
 
     def sanitize_uri(
@@ -207,15 +189,14 @@ class IndexDomainService:
     async def refresh_working_copy(
         self,
         working_copy: domain_entities.WorkingCopy,
-        progress_callback: ProgressCallback | None = None,
+        step: ProgressTracker | None = None,
     ) -> domain_entities.WorkingCopy:
         """Refresh the working copy."""
+        step = step or create_noop_operation()
         metadata_extractor = FileMetadataExtractor(working_copy.source_type)
-        reporter = Reporter(self.log, progress_callback)
-
         if working_copy.source_type == domain_entities.SourceType.GIT:
             git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
-            await git_working_copy_provider.sync(str(working_copy.remote_uri))
+            await git_working_copy_provider.sync(str(working_copy.remote_uri), step)
 
         current_file_paths = working_copy.list_filesystem_paths(
             GitIgnorePatternProvider(working_copy.cloned_path)
@@ -241,19 +222,12 @@ class IndexDomainService:
 
         # Setup reporter
         processed = 0
-        await reporter.start(
-            "refresh_working_copy", num_files_to_process, "Refreshing working copy..."
-        )
+        step.set_total(num_files_to_process)
 
         # First check to see if any files have been deleted
         for file_path in deleted_file_paths:
             processed += 1
-            await reporter.step(
-                "refresh_working_copy",
-                processed,
-                num_files_to_process,
-                f"Deleted {file_path.name}",
-            )
+            step.set_current(processed)
             previous_files_map[
                 file_path
             ].file_processing_status = domain_entities.FileProcessingStatus.DELETED
@@ -261,12 +235,7 @@ class IndexDomainService:
         # Then check to see if there are any new files
         for file_path in new_file_paths:
             processed += 1
-            await reporter.step(
-                "refresh_working_copy",
-                processed,
-                num_files_to_process,
-                f"New {file_path.name}",
-            )
+            step.set_current(processed)
             try:
                 working_copy.files.append(
                     await metadata_extractor.extract(file_path=file_path)
@@ -278,12 +247,7 @@ class IndexDomainService:
         # Finally check if there are any modified files
         for file_path in modified_file_paths:
             processed += 1
-            await reporter.step(
-                "refresh_working_copy",
-                processed,
-                num_files_to_process,
-                f"Modified {file_path.name}",
-            )
+            step.set_current(processed)
             try:
                 previous_file = previous_files_map[file_path]
                 new_file = await metadata_extractor.extract(file_path=file_path)

kodit/domain/value_objects.py

@@ -1,9 +1,9 @@
 """Pure domain value objects and DTOs."""
 
 import json
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import datetime
-from enum import Enum, IntEnum
+from enum import Enum, IntEnum, StrEnum
 from pathlib import Path
 from typing import ClassVar
 
@@ -390,18 +390,18 @@ class IndexRunRequest:
 
 
 @dataclass
-class ProgressEvent:
-    """Domain model for progress events."""
+class ProgressState:
+    """Progress state."""
 
-    operation: str
-    current: int
-    total: int
-    message: str | None = None
+    current: int = 0
+    total: int = 0
+    operation: str = ""
+    message: str = ""
 
     @property
     def percentage(self) -> float:
         """Calculate the percentage of completion."""
-        return (self.current / self.total * 100) if self.total > 0 else 0.0
+        return (self.current / self.total) * 100 if self.total > 0 else 0.0
 
 
 @dataclass
@@ -662,3 +662,51 @@ class QueuePriority(IntEnum):
 
     BACKGROUND = 10
     USER_INITIATED = 50
+
+
+# Reporting value objects
+
+
+class ReportingState(StrEnum):
+    """Reporting state."""
+
+    STARTED = "started"
+    IN_PROGRESS = "in_progress"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+
+@dataclass(frozen=True)
+class Progress:
+    """Immutable representation of a step's state."""
+
+    name: str
+    state: ReportingState
+    message: str = ""
+    error: BaseException | None = None
+    total: int = 0
+    current: int = 0
+
+    @property
+    def completion_percent(self) -> float:
+        """Calculate the percentage of completion."""
+        if self.total == 0:
+            return 0.0
+        return min(100.0, max(0.0, (self.current / self.total) * 100.0))
+
+    def with_error(self, error: BaseException) -> "Progress":
+        """Return a new snapshot with updated error."""
+        return replace(self, error=error)
+
+    def with_total(self, total: int) -> "Progress":
+        """Return a new snapshot with updated total."""
+        return replace(self, total=total)
+
+    def with_progress(self, current: int) -> "Progress":
+        """Return a new snapshot with updated progress."""
+        return replace(self, current=current)
+
+    def with_state(self, state: ReportingState, message: str = "") -> "Progress":
+        """Return a new snapshot with updated state."""
+        return replace(self, state=state, message=message)

kodit/infrastructure/api/v1/__init__.py

@@ -1,5 +1,5 @@
 """API v1 modules."""
 
-from .routers import indexes_router, search_router
+from .routers import indexes_router, queue_router, search_router
 
-__all__ = ["indexes_router", "search_router"]
+__all__ = ["indexes_router", "queue_router", "search_router"]

kodit/infrastructure/api/v1/dependencies.py

@@ -1,13 +1,13 @@
 """FastAPI dependencies for the REST API."""
 
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Callable
 from typing import Annotated, cast
 
 from fastapi import Depends, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.application.factories.code_indexing_factory import (
-    create_code_indexing_application_service,
+    create_server_code_indexing_application_service,
 )
 from kodit.application.services.code_indexing_application_service import (
     CodeIndexingApplicationService,
@@ -16,7 +16,7 @@ from kodit.application.services.queue_service import QueueService
 from kodit.config import AppContext
 from kodit.domain.services.index_query_service import IndexQueryService
 from kodit.infrastructure.indexing.fusion_service import ReciprocalRankFusionService
-from kodit.infrastructure.sqlalchemy.index_repository import SqlAlchemyIndexRepository
+from kodit.infrastructure.sqlalchemy.index_repository import create_index_repository
 
 
 def get_app_context(request: Request) -> AppContext:
@@ -42,12 +42,25 @@ async def get_db_session(
 DBSessionDep = Annotated[AsyncSession, Depends(get_db_session)]
 
 
+async def get_db_session_factory(
+    app_context: AppContextDep,
+) -> AsyncGenerator[Callable[[], AsyncSession], None]:
+    """Get database session dependency."""
+    db = await app_context.get_db()
+    yield db.session_factory
+
+
+DBSessionFactoryDep = Annotated[
+    Callable[[], AsyncSession], Depends(get_db_session_factory)
+]
+
+
 async def get_index_query_service(
-    session: DBSessionDep,
+    session_factory: DBSessionFactoryDep,
 ) -> IndexQueryService:
     """Get index query service dependency."""
     return IndexQueryService(
-        index_repository=SqlAlchemyIndexRepository(session=session),
+        index_repository=create_index_repository(session_factory=session_factory),
         fusion_service=ReciprocalRankFusionService(),
     )
 
@@ -58,11 +71,11 @@ IndexQueryServiceDep = Annotated[IndexQueryService, Depends(get_index_query_serv
 async def get_indexing_app_service(
     app_context: AppContextDep,
     session: DBSessionDep,
+    session_factory: DBSessionFactoryDep,
 ) -> CodeIndexingApplicationService:
     """Get indexing application service dependency."""
-    return create_code_indexing_application_service(
-        app_context=app_context,
-        session=session,
+    return create_server_code_indexing_application_service(
+        app_context, session, session_factory
     )
 
 
@@ -72,11 +85,11 @@ IndexingAppServiceDep = Annotated[
 
 
 async def get_queue_service(
-    session: DBSessionDep,
+    session_factory: DBSessionFactoryDep,
 ) -> QueueService:
     """Get queue service dependency."""
     return QueueService(
-        session=session,
+        session_factory=session_factory,
     )
 
 

kodit/infrastructure/api/v1/routers/__init__.py

@@ -1,6 +1,7 @@
 """API v1 routers."""
 
 from .indexes import router as indexes_router
+from .queue import router as queue_router
 from .search import router as search_router
 
-__all__ = ["indexes_router", "search_router"]
+__all__ = ["indexes_router", "queue_router", "search_router"]

kodit/infrastructure/api/v1/routers/queue.py

@@ -0,0 +1,76 @@
+"""Queue management router for the REST API."""
+
+from fastapi import APIRouter, Depends, HTTPException
+
+from kodit.domain.value_objects import TaskType
+from kodit.infrastructure.api.middleware.auth import api_key_auth
+from kodit.infrastructure.api.v1.dependencies import QueueServiceDep
+from kodit.infrastructure.api.v1.schemas.queue import (
+    TaskAttributes,
+    TaskData,
+    TaskListResponse,
+    TaskResponse,
+)
+
+router = APIRouter(
+    prefix="/api/v1/queue",
+    tags=["queue"],
+    dependencies=[Depends(api_key_auth)],
+    responses={
+        401: {"description": "Unauthorized"},
+        422: {"description": "Invalid request"},
+    },
+)
+
+
+@router.get("")
+async def list_queue_tasks(
+    queue_service: QueueServiceDep,
+    task_type: TaskType | None = None,
+) -> TaskListResponse:
+    """List all tasks in the queue.
+
+    Optionally filter by task type.
+    """
+    tasks = await queue_service.list_tasks(task_type)
+    return TaskListResponse(
+        data=[
+            TaskData(
+                type="task",
+                id=task.id,
+                attributes=TaskAttributes(
+                    type=str(task.type),
+                    priority=task.priority,
+                    payload=task.payload,
+                    created_at=task.created_at,
+                    updated_at=task.updated_at,
+                ),
+            )
+            for task in tasks
+        ]
+    )
+
+
+@router.get("/{task_id}", responses={404: {"description": "Task not found"}})
+async def get_queue_task(
+    task_id: str,
+    queue_service: QueueServiceDep,
+) -> TaskResponse:
+    """Get details of a specific task in the queue."""
+    task = await queue_service.get_task(task_id)
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+
+    return TaskResponse(
+        data=TaskData(
+            type="task",
+            id=task.id,
+            attributes=TaskAttributes(
+                type=str(task.type),
+                priority=task.priority,
+                payload=task.payload,
+                created_at=task.created_at,
+                updated_at=task.updated_at,
+            ),
+        )
+    )

kodit/infrastructure/api/v1/schemas/queue.py

@@ -0,0 +1,35 @@
+"""JSON:API schemas for queue operations."""
+
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class TaskAttributes(BaseModel):
+    """Task attributes for JSON:API responses."""
+
+    type: str
+    priority: int
+    payload: dict
+    created_at: datetime | None
+    updated_at: datetime | None
+
+
+class TaskData(BaseModel):
+    """Task data for JSON:API responses."""
+
+    type: str = "task"
+    id: str
+    attributes: TaskAttributes
+
+
+class TaskResponse(BaseModel):
+    """JSON:API response for single task."""
+
+    data: TaskData
+
+
+class TaskListResponse(BaseModel):
+    """JSON:API response for task list."""
+
+    data: list[TaskData]

kodit/infrastructure/cloning/git/working_copy.py

@@ -7,6 +7,8 @@ from pathlib import Path
 import git
 import structlog
 
+from kodit.application.factories.reporting_factory import create_noop_operation
+from kodit.application.services.reporting import ProgressTracker
 from kodit.domain.entities import WorkingCopy
 
 
@@ -25,18 +27,42 @@ class GitWorkingCopyProvider:
         dir_name = f"repo-{dir_hash}"
         return self.clone_dir / dir_name
 
-    async def prepare(self, uri: str) -> Path:
+    async def prepare(
+        self,
+        uri: str,
+        step: ProgressTracker | None = None,
+    ) -> Path:
         """Prepare a Git working copy."""
+        step = step or create_noop_operation()
         sanitized_uri = WorkingCopy.sanitize_git_url(uri)
         clone_path = self.get_clone_path(uri)
         clone_path.mkdir(parents=True, exist_ok=True)
 
+        step_record = []
+        step.set_total(12)
+
+        def _clone_progress_callback(
+            a: int, _: str | float | None, __: str | float | None, _d: str
+        ) -> None:
+            if a not in step_record:
+                step_record.append(a)
+
+            # Git reports a really weird format. This is a quick hack to get some
+            # progress.
+            step.set_current(len(step_record))
+
         try:
             self.log.info(
                 "Cloning repository", uri=sanitized_uri, clone_path=str(clone_path)
             )
             # Use the original URI for cloning (with credentials if present)
-            git.Repo.clone_from(uri, clone_path)
+            options = ["--depth=1", "--single-branch"]
+            git.Repo.clone_from(
+                uri,
+                clone_path,
+                progress=_clone_progress_callback,
+                multi_options=options,
+            )
         except git.GitCommandError as e:
             if "already exists and is not an empty directory" not in str(e):
                 msg = f"Failed to clone repository: {e}"
@@ -45,8 +71,9 @@ class GitWorkingCopyProvider:
 
         return clone_path
 
-    async def sync(self, uri: str) -> Path:
+    async def sync(self, uri: str, step: ProgressTracker | None = None) -> Path:
         """Refresh a Git working copy."""
+        step = step or create_noop_operation()
         clone_path = self.get_clone_path(uri)
 
         # Check if the clone directory exists and is a valid Git repository
@@ -54,9 +81,10 @@ class GitWorkingCopyProvider:
             self.log.info(
                 "Clone directory does not exist or is not a Git repository, "
                 "preparing...",
-                uri=uri, clone_path=str(clone_path)
+                uri=uri,
+                clone_path=str(clone_path),
             )
-            return await self.prepare(uri)
+            return await self.prepare(uri, step)
 
         try:
             repo = git.Repo(clone_path)
@@ -64,10 +92,11 @@ class GitWorkingCopyProvider:
         except git.InvalidGitRepositoryError:
             self.log.warning(
                 "Invalid Git repository found, re-cloning...",
-                uri=uri, clone_path=str(clone_path)
+                uri=uri,
+                clone_path=str(clone_path),
             )
             # Remove the invalid directory and re-clone
             shutil.rmtree(clone_path)
-            return await self.prepare(uri)
+            return await self.prepare(uri, step)
 
         return clone_path

kodit/infrastructure/embedding/embedding_factory.py

@@ -1,5 +1,8 @@
 """Factory for creating embedding services with DDD architecture."""
 
+from collections.abc import Callable
+
+import structlog
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.config import AppContext, Endpoint
@@ -8,14 +11,13 @@ from kodit.domain.services.embedding_service import (
     EmbeddingProvider,
     VectorSearchRepository,
 )
+from kodit.infrastructure.embedding.embedding_providers.litellm_embedding_provider import (  # noqa: E501
+    LiteLLMEmbeddingProvider,
+)
 from kodit.infrastructure.embedding.embedding_providers.local_embedding_provider import (  # noqa: E501
     CODE,
     LocalEmbeddingProvider,
 )
-from kodit.infrastructure.embedding.embedding_providers.openai_embedding_provider import (  # noqa: E501
-    OPENAI_NUM_PARALLEL_TASKS,
-    OpenAIEmbeddingProvider,
-)
 from kodit.infrastructure.embedding.local_vector_search_repository import (
     LocalVectorSearchRepository,
 )
@@ -24,7 +26,7 @@ from kodit.infrastructure.embedding.vectorchord_vector_search_repository import
     VectorChordVectorSearchRepository,
 )
 from kodit.infrastructure.sqlalchemy.embedding_repository import (
-    SqlAlchemyEmbeddingRepository,
+    create_embedding_repository,
 )
 from kodit.infrastructure.sqlalchemy.entities import EmbeddingType
 from kodit.log import log_event
@@ -32,30 +34,27 @@ from kodit.log import log_event
 
 def _get_endpoint_configuration(app_context: AppContext) -> Endpoint | None:
     """Get the endpoint configuration for the embedding service."""
-    return app_context.embedding_endpoint or app_context.default_endpoint or None
+    return app_context.embedding_endpoint or None
 
 
 def embedding_domain_service_factory(
-    task_name: TaskName, app_context: AppContext, session: AsyncSession
+    task_name: TaskName,
+    app_context: AppContext,
+    session: AsyncSession,
+    session_factory: Callable[[], AsyncSession],
 ) -> EmbeddingDomainService:
     """Create an embedding domain service."""
+    structlog.get_logger(__name__)
     # Create embedding repository
-    embedding_repository = SqlAlchemyEmbeddingRepository(session=session)
+    embedding_repository = create_embedding_repository(session_factory=session_factory)
 
     # Create embedding provider
     embedding_provider: EmbeddingProvider | None = None
     endpoint = _get_endpoint_configuration(app_context)
-    if endpoint and endpoint.type == "openai":
-        log_event("kodit.embedding", {"provider": "openai"})
-        # Use new httpx-based provider with socket support
-        embedding_provider = OpenAIEmbeddingProvider(
-            api_key=endpoint.api_key,
-            base_url=endpoint.base_url or "https://api.openai.com/v1",
-            model_name=endpoint.model or "text-embedding-3-small",
-            num_parallel_tasks=endpoint.num_parallel_tasks or OPENAI_NUM_PARALLEL_TASKS,
-            socket_path=endpoint.socket_path,
-            timeout=endpoint.timeout or 30.0,
-        )
+
+    if endpoint:
+        log_event("kodit.embedding", {"provider": "litellm"})
+        embedding_provider = LiteLLMEmbeddingProvider(endpoint=endpoint)
     else:
         log_event("kodit.embedding", {"provider": "local"})
         embedding_provider = LocalEmbeddingProvider(CODE)