kodit 0.4.3__py3-none-any.whl → 0.5.1__py3-none-any.whl

kodit/domain/services/index_service.py

@@ -1,269 +0,0 @@
-"""Pure domain service for Index aggregate operations."""
-
-from abc import ABC, abstractmethod
-from collections import defaultdict
-from pathlib import Path
-
-import structlog
-from pydantic import AnyUrl
-
-import kodit.domain.entities as domain_entities
-from kodit.application.factories.reporting_factory import create_noop_operation
-from kodit.application.services.reporting import ProgressTracker
-from kodit.domain.services.enrichment_service import EnrichmentDomainService
-from kodit.domain.value_objects import (
-    EnrichmentIndexRequest,
-    EnrichmentRequest,
-    FileProcessingStatus,
-    LanguageMapping,
-)
-from kodit.infrastructure.cloning.git.working_copy import GitWorkingCopyProvider
-from kodit.infrastructure.cloning.metadata import FileMetadataExtractor
-from kodit.infrastructure.git.git_utils import is_valid_clone_target
-from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
-from kodit.infrastructure.slicing.slicer import Slicer
-from kodit.utils.path_utils import path_from_uri
-
-
-class LanguageDetectionService(ABC):
-    """Abstract interface for language detection service."""
-
-    @abstractmethod
-    async def detect_language(self, file_path: Path) -> str:
-        """Detect the programming language of a file."""
-
-
-class IndexDomainService:
-    """Pure domain service for Index aggregate operations.
-
-    This service handles the full lifecycle of code indexing:
-    - Creating indexes for source repositories
-    - Cloning and processing source files
-    - Extracting and enriching code snippets
-    - Managing the complete Index aggregate
-    """
-
-    def __init__(
-        self,
-        language_detector: LanguageDetectionService,
-        enrichment_service: EnrichmentDomainService,
-        clone_dir: Path,
-    ) -> None:
-        """Initialize the index domain service."""
-        self._clone_dir = clone_dir
-        self._language_detector = language_detector
-        self._enrichment_service = enrichment_service
-        self.log = structlog.get_logger(__name__)
-
-    async def prepare_index(
-        self,
-        uri_or_path_like: str,  # Must include user/pass, etc
-        step: ProgressTracker | None = None,
-    ) -> domain_entities.WorkingCopy:
-        """Prepare an index by scanning files and creating working copy."""
-        step = step or create_noop_operation()
-        self.log.info("Preparing index")
-        sanitized_uri, source_type = self.sanitize_uri(uri_or_path_like)
-        self.log.info("Preparing source", uri=str(sanitized_uri))
-
-        if source_type == domain_entities.SourceType.FOLDER:
-            local_path = path_from_uri(str(sanitized_uri))
-        elif source_type == domain_entities.SourceType.GIT:
-            source_type = domain_entities.SourceType.GIT
-            git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
-            local_path = await git_working_copy_provider.prepare(uri_or_path_like, step)
-        else:
-            raise ValueError(f"Unsupported source: {uri_or_path_like}")
-
-        return domain_entities.WorkingCopy(
-            remote_uri=sanitized_uri,
-            cloned_path=local_path,
-            source_type=source_type,
-            files=[],
-        )
-
-    async def extract_snippets_from_index(
-        self,
-        index: domain_entities.Index,
-        step: ProgressTracker | None = None,
-    ) -> domain_entities.Index:
-        """Extract code snippets from files in the index."""
-        step = step or create_noop_operation()
-        file_count = len(index.source.working_copy.files)
-
-        self.log.info(
-            "Extracting snippets",
-            index_id=index.id,
-            file_count=file_count,
-        )
-
-        # Only create snippets for files that have been added or modified
-        files = index.source.working_copy.changed_files()
-        index.delete_snippets_for_files(files)
-
-        # Filter out deleted files - they don't exist on disk anymore
-        files = [
-            f for f in files if f.file_processing_status != FileProcessingStatus.DELETED
-        ]
-
-        # Create a set of languages to extract snippets for
-        extensions = {file.extension() for file in files}
-        lang_files_map: dict[str, list[domain_entities.File]] = defaultdict(list)
-        for ext in extensions:
-            try:
-                lang = LanguageMapping.get_language_for_extension(ext)
-                lang_files_map[lang].extend(
-                    file for file in files if file.extension() == ext
-                )
-            except ValueError as e:
-                self.log.debug("Skipping", error=str(e))
-                continue
-
-        self.log.info(
-            "Languages to process",
-            languages=lang_files_map.keys(),
-        )
-
-        # Calculate snippets for each language
-        slicer = Slicer()
-        await step.set_total(len(lang_files_map.keys()))
-        for i, (lang, lang_files) in enumerate(lang_files_map.items()):
-            await step.set_current(i, f"Extracting snippets for {lang}")
-            s = slicer.extract_snippets(lang_files, language=lang)
-            index.snippets.extend(s)
-
-        return index
-
-    async def enrich_snippets_in_index(
-        self,
-        snippets: list[domain_entities.Snippet],
-        reporting_step: ProgressTracker | None = None,
-    ) -> list[domain_entities.Snippet]:
-        """Enrich snippets with AI-generated summaries."""
-        reporting_step = reporting_step or create_noop_operation()
-        if not snippets or len(snippets) == 0:
-            await reporting_step.skip("No snippets to enrich")
-            return snippets
-
-        await reporting_step.set_total(len(snippets))
-        snippet_map = {snippet.id: snippet for snippet in snippets if snippet.id}
-
-        enrichment_request = EnrichmentIndexRequest(
-            requests=[
-                EnrichmentRequest(snippet_id=snippet_id, text=snippet.original_text())
-                for snippet_id, snippet in snippet_map.items()
-            ]
-        )
-
-        processed = 0
-        async for result in self._enrichment_service.enrich_documents(
-            enrichment_request
-        ):
-            snippet_map[result.snippet_id].add_summary(result.text)
-
-            processed += 1
-            await reporting_step.set_current(
-                processed, f"Enriching snippets for {processed} snippets"
-            )
-
-        return list(snippet_map.values())
-
-    def sanitize_uri(
-        self, uri_or_path_like: str
-    ) -> tuple[AnyUrl, domain_entities.SourceType]:
-        """Convert a URI or path-like string to a URI."""
-        # First, check if it's a local directory (more reliable than git check)
-        if Path(uri_or_path_like).is_dir():
-            return (
-                domain_entities.WorkingCopy.sanitize_local_path(uri_or_path_like),
-                domain_entities.SourceType.FOLDER,
-            )
-
-        # Then check if it's git-clonable
-        if is_valid_clone_target(uri_or_path_like):
-            return (
-                domain_entities.WorkingCopy.sanitize_git_url(uri_or_path_like),
-                domain_entities.SourceType.GIT,
-            )
-
-        raise ValueError(f"Unsupported source: {uri_or_path_like}")
-
-    async def refresh_working_copy(
-        self,
-        working_copy: domain_entities.WorkingCopy,
-        step: ProgressTracker | None = None,
-    ) -> domain_entities.WorkingCopy:
-        """Refresh the working copy."""
-        step = step or create_noop_operation()
-        metadata_extractor = FileMetadataExtractor(working_copy.source_type)
-        if working_copy.source_type == domain_entities.SourceType.GIT:
-            git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
-            await git_working_copy_provider.sync(str(working_copy.remote_uri), step)
-
-        current_file_paths = working_copy.list_filesystem_paths(
-            GitIgnorePatternProvider(working_copy.cloned_path)
-        )
-
-        previous_files_map = {file.as_path(): file for file in working_copy.files}
-
-        # Calculate different sets of files
-        deleted_file_paths = set(previous_files_map.keys()) - set(current_file_paths)
-        new_file_paths = set(current_file_paths) - set(previous_files_map.keys())
-        modified_file_paths = set(current_file_paths) & set(previous_files_map.keys())
-        num_files_to_process = (
-            len(deleted_file_paths) + len(new_file_paths) + len(modified_file_paths)
-        )
-        self.log.info(
-            "Refreshing working copy",
-            num_deleted=len(deleted_file_paths),
-            num_new=len(new_file_paths),
-            num_modified=len(modified_file_paths),
-            num_total_changes=num_files_to_process,
-            num_dirty=len(working_copy.dirty_files()),
-        )
-
-        # Setup reporter
-        processed = 0
-        await step.set_total(num_files_to_process)
-
-        # First check to see if any files have been deleted
-        for file_path in deleted_file_paths:
-            processed += 1
-            await step.set_current(processed, f"Deleting file {file_path}")
-            previous_files_map[
-                file_path
-            ].file_processing_status = domain_entities.FileProcessingStatus.DELETED
-
-        # Then check to see if there are any new files
-        for file_path in new_file_paths:
-            processed += 1
-            await step.set_current(processed, f"Adding new file {file_path}")
-            try:
-                working_copy.files.append(
-                    await metadata_extractor.extract(file_path=file_path)
-                )
-            except (OSError, ValueError) as e:
-                self.log.debug("Skipping file", file=str(file_path), error=str(e))
-                continue
-
-        # Finally check if there are any modified files
-        for file_path in modified_file_paths:
-            processed += 1
-            await step.set_current(processed, f"Modifying file {file_path}")
-            try:
-                previous_file = previous_files_map[file_path]
-                new_file = await metadata_extractor.extract(file_path=file_path)
-                if previous_file.sha256 != new_file.sha256:
-                    previous_file.file_processing_status = (
-                        domain_entities.FileProcessingStatus.MODIFIED
-                    )
-            except (OSError, ValueError) as e:
-                self.log.info("Skipping file", file=str(file_path), error=str(e))
-                continue
-
-        return working_copy
-
-    async def delete_index(self, index: domain_entities.Index) -> None:
-        """Delete an index."""
-        # Delete the working copy
-        index.source.working_copy.delete()

kodit/infrastructure/api/client/index_client.py

@@ -1,57 +0,0 @@
-"""Index operations API client for Kodit server."""
-
-from kodit.infrastructure.api.v1.schemas.index import (
-    IndexCreateAttributes,
-    IndexCreateData,
-    IndexCreateRequest,
-    IndexData,
-    IndexListResponse,
-    IndexResponse,
-)
-
-from .base import BaseAPIClient
-from .exceptions import KoditAPIError
-from .generated_endpoints import APIEndpoints
-
-
-class IndexClient(BaseAPIClient):
-    """API client for index operations."""
-
-    async def list_indexes(self) -> list[IndexData]:
-        """List all indexes."""
-        response = await self._request("GET", APIEndpoints.API_V1_INDEXES)
-        data = IndexListResponse.model_validate_json(response.text)
-        return data.data
-
-    async def create_index(self, uri: str) -> IndexData:
-        """Create a new index."""
-        request = IndexCreateRequest(
-            data=IndexCreateData(
-                type="index", attributes=IndexCreateAttributes(uri=uri)
-            )
-        )
-        response = await self._request(
-            "POST", APIEndpoints.API_V1_INDEXES, json=request.model_dump()
-        )
-        result = IndexResponse.model_validate_json(response.text)
-        return result.data
-
-    async def get_index(self, index_id: str) -> IndexData | None:
-        """Get index by ID."""
-        try:
-            response = await self._request(
-                "GET", APIEndpoints.API_V1_INDEXES_INDEX_ID.format(index_id=index_id)
-            )
-            result = IndexResponse.model_validate_json(response.text)
-        except KoditAPIError as e:
-            if "404" in str(e):
-                return None
-            raise
-        else:
-            return result.data
-
-    async def delete_index(self, index_id: str) -> None:
-        """Delete an index."""
-        await self._request(
-            "DELETE", APIEndpoints.API_V1_INDEXES_INDEX_ID.format(index_id=index_id)
-        )

kodit/infrastructure/api/v1/routers/indexes.py

@@ -1,164 +0,0 @@
-"""Index management router for the REST API."""
-
-from fastapi import APIRouter, Depends, HTTPException
-
-from kodit.domain.entities import Task
-from kodit.domain.value_objects import QueuePriority
-from kodit.infrastructure.api.middleware.auth import api_key_auth
-from kodit.infrastructure.api.v1.dependencies import (
-    IndexingAppServiceDep,
-    IndexQueryServiceDep,
-    QueueServiceDep,
-    TaskStatusQueryServiceDep,
-)
-from kodit.infrastructure.api.v1.schemas.index import (
-    IndexAttributes,
-    IndexCreateRequest,
-    IndexData,
-    IndexDetailResponse,
-    IndexListResponse,
-    IndexResponse,
-)
-from kodit.infrastructure.api.v1.schemas.task_status import (
-    TaskStatusAttributes,
-    TaskStatusData,
-    TaskStatusListResponse,
-)
-
-router = APIRouter(
-    prefix="/api/v1/indexes",
-    tags=["indexes"],
-    dependencies=[Depends(api_key_auth)],
-    responses={
-        401: {"description": "Unauthorized"},
-        422: {"description": "Invalid request"},
-    },
-)
-
-
-@router.get("")
-async def list_indexes(
-    query_service: IndexQueryServiceDep,
-) -> IndexListResponse:
-    """List all indexes."""
-    indexes = await query_service.list_indexes()
-    return IndexListResponse(
-        data=[
-            IndexData(
-                type="index",
-                id=str(idx.id),
-                attributes=IndexAttributes(
-                    created_at=idx.created_at,
-                    updated_at=idx.updated_at,
-                    uri=str(idx.source.working_copy.remote_uri),
-                ),
-            )
-            for idx in indexes
-        ]
-    )
-
-
-@router.post("", status_code=202)
-async def create_index(
-    request: IndexCreateRequest,
-    app_service: IndexingAppServiceDep,
-    queue_service: QueueServiceDep,
-) -> IndexResponse:
-    """Create a new index and start async indexing."""
-    # Create index using the application service
-    index = await app_service.create_index_from_uri(request.data.attributes.uri)
-
-    # Add the indexing task to the queue
-    await queue_service.enqueue_task(
-        Task.create_index_update_task(index.id, QueuePriority.USER_INITIATED)
-    )
-
-    return IndexResponse(
-        data=IndexData(
-            type="index",
-            id=str(index.id),
-            attributes=IndexAttributes(
-                created_at=index.created_at,
-                updated_at=index.updated_at,
-                uri=str(index.source.working_copy.remote_uri),
-            ),
-        )
-    )
-
-
-@router.get("/{index_id}", responses={404: {"description": "Index not found"}})
-async def get_index(
-    index_id: int,
-    query_service: IndexQueryServiceDep,
-) -> IndexDetailResponse:
-    """Get index details."""
-    index = await query_service.get_index_by_id(index_id)
-    if not index:
-        raise HTTPException(status_code=404, detail="Index not found")
-
-    return IndexDetailResponse(
-        data=IndexData(
-            type="index",
-            id=str(index.id),
-            attributes=IndexAttributes(
-                created_at=index.created_at,
-                updated_at=index.updated_at,
-                uri=str(index.source.working_copy.remote_uri),
-            ),
-        ),
-    )
-
-
-@router.get(
-    "/{index_id}/status",
-    responses={404: {"description": "Index not found"}},
-)
-async def get_index_status(
-    index_id: int,
-    query_service: IndexQueryServiceDep,
-    status_service: TaskStatusQueryServiceDep,
-) -> TaskStatusListResponse:
-    """Get the status of tasks for an index."""
-    # Verify the index exists
-    index = await query_service.get_index_by_id(index_id)
-    if not index:
-        raise HTTPException(status_code=404, detail="Index not found")
-
-    # Get all task statuses for this index
-    progress_trackers = await status_service.get_index_status(index_id)
-
-    # Convert progress trackers to API response format
-    task_statuses = []
-    for _i, status in enumerate(progress_trackers):
-        task_statuses.append(
-            TaskStatusData(
-                id=status.id,
-                attributes=TaskStatusAttributes(
-                    step=status.operation,
-                    state=status.state,
-                    progress=status.completion_percent,
-                    total=status.total,
-                    current=status.current,
-                    created_at=status.created_at,
-                    updated_at=status.updated_at,
-                ),
-            )
-        )
-
-    return TaskStatusListResponse(data=task_statuses)
-
-
-@router.delete(
-    "/{index_id}", status_code=204, responses={404: {"description": "Index not found"}}
-)
-async def delete_index(
-    index_id: int,
-    query_service: IndexQueryServiceDep,
-    app_service: IndexingAppServiceDep,
-) -> None:
-    """Delete an index."""
-    index = await query_service.get_index_by_id(index_id)
-    if not index:
-        raise HTTPException(status_code=404, detail="Index not found")
-
-    await app_service.delete_index(index)

kodit/infrastructure/api/v1/schemas/index.py

@@ -1,101 +0,0 @@
-"""JSON:API schemas for index operations."""
-
-from datetime import datetime
-
-from pydantic import BaseModel, Field
-
-
-class IndexAttributes(BaseModel):
-    """Index attributes for JSON:API responses."""
-
-    created_at: datetime
-    updated_at: datetime
-    uri: str
-
-
-class SnippetData(BaseModel):
-    """Snippet data for JSON:API relationships."""
-
-    type: str = "snippet"
-    id: str
-
-
-class IndexData(BaseModel):
-    """Index data for JSON:API responses."""
-
-    type: str = "index"
-    id: str
-    attributes: IndexAttributes
-
-
-class IndexResponse(BaseModel):
-    """JSON:API response for single index."""
-
-    data: IndexData
-
-
-class IndexListResponse(BaseModel):
-    """JSON:API response for index list."""
-
-    data: list[IndexData]
-
-
-class IndexCreateAttributes(BaseModel):
-    """Attributes for creating an index."""
-
-    uri: str = Field(..., description="URI of the source to index")
-
-
-class IndexCreateData(BaseModel):
-    """Data for creating an index."""
-
-    type: str = "index"
-    attributes: IndexCreateAttributes
-
-
-class IndexCreateRequest(BaseModel):
-    """JSON:API request for creating an index."""
-
-    data: IndexCreateData
-
-
-class AuthorData(BaseModel):
-    """Author data for JSON:API relationships."""
-
-    type: str = "author"
-    id: str
-
-
-class AuthorsRelationship(BaseModel):
-    """Authors relationship for JSON:API."""
-
-    data: list[AuthorData]
-
-
-class FileRelationships(BaseModel):
-    """File relationships for JSON:API."""
-
-    authors: AuthorsRelationship
-
-
-class FileAttributes(BaseModel):
-    """File attributes for JSON:API included resources."""
-
-    uri: str
-    sha256: str
-    mime_type: str
-    created_at: datetime
-    updated_at: datetime
-
-
-class AuthorAttributes(BaseModel):
-    """Author attributes for JSON:API included resources."""
-
-    name: str
-    email: str
-
-
-class IndexDetailResponse(BaseModel):
-    """JSON:API response for index details with included resources."""
-
-    data: IndexData

kodit/infrastructure/bm25/bm25_factory.py

@@ -1,28 +0,0 @@
-"""Factory for creating BM25 repositories."""
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from kodit.config import AppContext
-from kodit.domain.services.bm25_service import BM25Repository
-from kodit.infrastructure.bm25.local_bm25_repository import LocalBM25Repository
-from kodit.infrastructure.bm25.vectorchord_bm25_repository import (
-    VectorChordBM25Repository,
-)
-
-
-def bm25_repository_factory(
-    app_context: AppContext, session: AsyncSession
-) -> BM25Repository:
-    """Create a BM25 repository based on configuration.
-
-    Args:
-        app_context: Application configuration context
-        session: SQLAlchemy async session
-
-    Returns:
-        BM25Repository instance
-
-    """
-    if app_context.default_search.provider == "vectorchord":
-        return VectorChordBM25Repository(session=session)
-    return LocalBM25Repository(data_dir=app_context.get_data_dir())

kodit/infrastructure/cloning/__init__.py

@@ -1 +0,0 @@
-"""Cloning infrastructure."""