kodit 0.3.2__py3-none-any.whl → 0.3.3__py3-none-any.whl

kodit/domain/services/embedding_service.py

@@ -3,7 +3,6 @@
 from abc import ABC, abstractmethod
 from collections.abc import AsyncGenerator, Sequence
 
-from kodit.domain.entities import EmbeddingType
 from kodit.domain.value_objects import (
     EmbeddingRequest,
     EmbeddingResponse,
@@ -12,6 +11,7 @@ from kodit.domain.value_objects import (
     SearchRequest,
     SearchResult,
 )
+from kodit.infrastructure.sqlalchemy.entities import EmbeddingType
 
 
 class EmbeddingProvider(ABC):

kodit/domain/services/index_query_service.py

@@ -0,0 +1,66 @@
+"""Index query service."""
+
+from abc import ABC, abstractmethod
+
+from kodit.domain.entities import Index, SnippetWithContext
+from kodit.domain.protocols import IndexRepository
+from kodit.domain.value_objects import (
+    FusionRequest,
+    FusionResult,
+    MultiSearchRequest,
+)
+
+
+class FusionService(ABC):
+    """Abstract fusion service interface."""
+
+    @abstractmethod
+    def reciprocal_rank_fusion(
+        self, rankings: list[list[FusionRequest]], k: float = 60
+    ) -> list[FusionResult]:
+        """Perform reciprocal rank fusion on search results."""
+
+
+class IndexQueryService:
+    """Index query service."""
+
+    def __init__(
+        self,
+        index_repository: IndexRepository,
+        fusion_service: FusionService,
+    ) -> None:
+        """Initialize the index query service."""
+        self.index_repository = index_repository
+        self.fusion_service = fusion_service
+
+    async def get_index_by_id(self, index_id: int) -> Index | None:
+        """Get an index by its ID."""
+        return await self.index_repository.get(index_id)
+
+    async def list_indexes(self) -> list[Index]:
+        """List all indexes."""
+        return await self.index_repository.all()
+
+    async def search_snippets(
+        self, request: MultiSearchRequest
+    ) -> list[SnippetWithContext]:
+        """Search snippets with filters.
+
+        Args:
+            request: The search request containing filters
+
+        Returns:
+            List of matching snippet items with context
+
+        """
+        return list(await self.index_repository.search(request))
+
+    async def perform_fusion(
+        self, rankings: list[list[FusionRequest]], k: float = 60
+    ) -> list[FusionResult]:
+        """Perform reciprocal rank fusion on search results."""
+        return self.fusion_service.reciprocal_rank_fusion(rankings, k)
+
+    async def get_snippets_by_ids(self, ids: list[int]) -> list[SnippetWithContext]:
+        """Get snippets by their IDs."""
+        return await self.index_repository.get_snippets_by_ids(ids)

kodit/domain/services/index_service.py

@@ -0,0 +1,323 @@
+"""Pure domain service for Index aggregate operations."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from pathlib import Path
+
+import structlog
+from pydantic import AnyUrl
+
+import kodit.domain.entities as domain_entities
+from kodit.domain.interfaces import ProgressCallback
+from kodit.domain.services.enrichment_service import EnrichmentDomainService
+from kodit.domain.value_objects import (
+    EnrichmentIndexRequest,
+    EnrichmentRequest,
+    SnippetExtractionRequest,
+    SnippetExtractionResult,
+    SnippetExtractionStrategy,
+)
+from kodit.infrastructure.cloning.git.working_copy import GitWorkingCopyProvider
+from kodit.infrastructure.cloning.metadata import FileMetadataExtractor
+from kodit.infrastructure.git.git_utils import is_valid_clone_target
+from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
+from kodit.reporting import Reporter
+from kodit.utils.path_utils import path_from_uri
+
+
+class LanguageDetectionService(ABC):
+    """Abstract interface for language detection service."""
+
+    @abstractmethod
+    async def detect_language(self, file_path: Path) -> str:
+        """Detect the programming language of a file."""
+
+
+class SnippetExtractor(ABC):
+    """Abstract interface for snippet extraction."""
+
+    @abstractmethod
+    async def extract(self, file_path: Path, language: str) -> list[str]:
+        """Extract snippets from a file."""
+
+
+class IndexDomainService:
+    """Pure domain service for Index aggregate operations.
+
+    This service handles the full lifecycle of code indexing:
+    - Creating indexes for source repositories
+    - Cloning and processing source files
+    - Extracting and enriching code snippets
+    - Managing the complete Index aggregate
+    """
+
+    def __init__(
+        self,
+        language_detector: LanguageDetectionService,
+        snippet_extractors: Mapping[SnippetExtractionStrategy, SnippetExtractor],
+        enrichment_service: EnrichmentDomainService,
+        clone_dir: Path,
+    ) -> None:
+        """Initialize the index domain service."""
+        self._clone_dir = clone_dir
+        self._language_detector = language_detector
+        self._snippet_extractors = snippet_extractors
+        self._enrichment_service = enrichment_service
+        self.log = structlog.get_logger(__name__)
+
+    async def prepare_index(
+        self,
+        uri_or_path_like: str,  # Must include user/pass, etc
+        progress_callback: ProgressCallback | None = None,
+    ) -> domain_entities.WorkingCopy:
+        """Prepare an index by scanning files and creating working copy."""
+        sanitized_uri, source_type = self.sanitize_uri(uri_or_path_like)
+        reporter = Reporter(self.log, progress_callback)
+        self.log.info("Preparing source", uri=str(sanitized_uri))
+
+        if source_type == domain_entities.SourceType.FOLDER:
+            await reporter.start("prepare_index", 1, "Scanning source...")
+            local_path = path_from_uri(str(sanitized_uri))
+        elif source_type == domain_entities.SourceType.GIT:
+            source_type = domain_entities.SourceType.GIT
+            git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
+            await reporter.start("prepare_index", 1, "Cloning source...")
+            local_path = await git_working_copy_provider.prepare(uri_or_path_like)
+            await reporter.done("prepare_index")
+        else:
+            raise ValueError(f"Unsupported source: {uri_or_path_like}")
+
+        await reporter.done("prepare_index")
+
+        return domain_entities.WorkingCopy(
+            remote_uri=sanitized_uri,
+            cloned_path=local_path,
+            source_type=source_type,
+            files=[],
+        )
+
+    async def extract_snippets_from_index(
+        self,
+        index: domain_entities.Index,
+        strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED,
+        progress_callback: ProgressCallback | None = None,
+    ) -> domain_entities.Index:
+        """Extract code snippets from files in the index."""
+        file_count = len(index.source.working_copy.files)
+
+        self.log.info(
+            "Extracting snippets",
+            index_id=index.id,
+            file_count=file_count,
+            strategy=strategy.value,
+        )
+
+        # Only create snippets for files that have been added or modified
+        files = index.source.working_copy.changed_files()
+        index.delete_snippets_for_files(files)
+
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "extract_snippets", len(files), "Extracting code snippets..."
+        )
+
+        new_snippets = []
+        for i, domain_file in enumerate(files, 1):
+            try:
+                # Extract snippets from file
+                request = SnippetExtractionRequest(
+                    file_path=domain_file.as_path(), strategy=strategy
+                )
+                result = await self._extract_snippets(request)
+                for snippet_text in result.snippets:
+                    snippet = domain_entities.Snippet(
+                        derives_from=[domain_file],
+                    )
+                    snippet.add_original_content(snippet_text, result.language)
+                    new_snippets.append(snippet)
+
+            except (OSError, ValueError) as e:
+                self.log.debug(
+                    "Skipping file for snippet extraction",
+                    file_uri=str(domain_file.uri),
+                    error=str(e),
+                )
+                continue
+
+            await reporter.step(
+                "extract_snippets", i, len(files), f"Processed {domain_file.uri.path}"
+            )
+
+        index.snippets.extend(new_snippets)
+        await reporter.done("extract_snippets")
+        return index
+
+    async def enrich_snippets_in_index(
+        self,
+        snippets: list[domain_entities.Snippet],
+        progress_callback: ProgressCallback | None = None,
+    ) -> list[domain_entities.Snippet]:
+        """Enrich snippets with AI-generated summaries."""
+        if not snippets or len(snippets) == 0:
+            return snippets
+
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start("enrichment", len(snippets), "Enriching snippets...")
+
+        snippet_map = {snippet.id: snippet for snippet in snippets if snippet.id}
+
+        enrichment_request = EnrichmentIndexRequest(
+            requests=[
+                EnrichmentRequest(snippet_id=snippet_id, text=snippet.original_text())
+                for snippet_id, snippet in snippet_map.items()
+            ]
+        )
+
+        processed = 0
+        async for result in self._enrichment_service.enrich_documents(
+            enrichment_request
+        ):
+            snippet_map[result.snippet_id].add_summary(result.text)
+
+            processed += 1
+            await reporter.step(
+                "enrichment", processed, len(snippets), "Enriching snippets..."
+            )
+
+        await reporter.done("enrichment")
+        return list(snippet_map.values())
+
+    async def _extract_snippets(
+        self, request: SnippetExtractionRequest
+    ) -> SnippetExtractionResult:
+        # Domain logic: validate file exists
+        if not request.file_path.exists():
+            raise ValueError(f"File does not exist: {request.file_path}")
+
+        # Domain logic: detect language
+        language = await self._language_detector.detect_language(request.file_path)
+
+        # Domain logic: choose strategy and extractor
+        if request.strategy not in self._snippet_extractors:
+            raise ValueError(f"Unsupported extraction strategy: {request.strategy}")
+
+        extractor = self._snippet_extractors[request.strategy]
+        snippets = await extractor.extract(request.file_path, language)
+
+        # Domain logic: filter out empty snippets
+        filtered_snippets = [snippet for snippet in snippets if snippet.strip()]
+
+        return SnippetExtractionResult(snippets=filtered_snippets, language=language)
+
+    def sanitize_uri(
+        self, uri_or_path_like: str
+    ) -> tuple[AnyUrl, domain_entities.SourceType]:
+        """Convert a URI or path-like string to a URI."""
+        # First, check if it's a local directory (more reliable than git check)
+        if Path(uri_or_path_like).is_dir():
+            return (
+                domain_entities.WorkingCopy.sanitize_local_path(uri_or_path_like),
+                domain_entities.SourceType.FOLDER,
+            )
+
+        # Then check if it's git-clonable
+        if is_valid_clone_target(uri_or_path_like):
+            return (
+                domain_entities.WorkingCopy.sanitize_git_url(uri_or_path_like),
+                domain_entities.SourceType.GIT,
+            )
+
+        raise ValueError(f"Unsupported source: {uri_or_path_like}")
+
+    async def refresh_working_copy(
+        self,
+        working_copy: domain_entities.WorkingCopy,
+        progress_callback: ProgressCallback | None = None,
+    ) -> domain_entities.WorkingCopy:
+        """Refresh the working copy."""
+        metadata_extractor = FileMetadataExtractor(working_copy.source_type)
+        reporter = Reporter(self.log, progress_callback)
+
+        if working_copy.source_type == domain_entities.SourceType.GIT:
+            git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
+            await git_working_copy_provider.sync(str(working_copy.remote_uri))
+
+        current_file_paths = working_copy.list_filesystem_paths(
+            GitIgnorePatternProvider(working_copy.cloned_path)
+        )
+
+        previous_files_map = {file.as_path(): file for file in working_copy.files}
+
+        # Calculate different sets of files
+        deleted_file_paths = set(previous_files_map.keys()) - set(current_file_paths)
+        new_file_paths = set(current_file_paths) - set(previous_files_map.keys())
+        modified_file_paths = set(current_file_paths) & set(previous_files_map.keys())
+        num_files_to_process = (
+            len(deleted_file_paths) + len(new_file_paths) + len(modified_file_paths)
+        )
+        self.log.info(
+            "Refreshing working copy",
+            num_deleted=len(deleted_file_paths),
+            num_new=len(new_file_paths),
+            num_modified=len(modified_file_paths),
+            num_total_changes=num_files_to_process,
+            num_dirty=len(working_copy.dirty_files()),
+        )
+
+        # Setup reporter
+        processed = 0
+        await reporter.start(
+            "refresh_working_copy", num_files_to_process, "Refreshing working copy..."
+        )
+
+        # First check to see if any files have been deleted
+        for file_path in deleted_file_paths:
+            processed += 1
+            await reporter.step(
+                "refresh_working_copy",
+                processed,
+                num_files_to_process,
+                f"Deleted {file_path.name}",
+            )
+            previous_files_map[
+                file_path
+            ].file_processing_status = domain_entities.FileProcessingStatus.DELETED
+
+        # Then check to see if there are any new files
+        for file_path in new_file_paths:
+            processed += 1
+            await reporter.step(
+                "refresh_working_copy",
+                processed,
+                num_files_to_process,
+                f"New {file_path.name}",
+            )
+            try:
+                working_copy.files.append(
+                    await metadata_extractor.extract(file_path=file_path)
+                )
+            except (OSError, ValueError) as e:
+                self.log.info("Skipping file", file=str(file_path), error=str(e))
+                continue
+
+        # Finally check if there are any modified files
+        for file_path in modified_file_paths:
+            processed += 1
+            await reporter.step(
+                "refresh_working_copy",
+                processed,
+                num_files_to_process,
+                f"Modified {file_path.name}",
+            )
+            try:
+                previous_file = previous_files_map[file_path]
+                new_file = await metadata_extractor.extract(file_path=file_path)
+                if previous_file.sha256 != new_file.sha256:
+                    previous_file.file_processing_status = (
+                        domain_entities.FileProcessingStatus.MODIFIED
+                    )
+            except (OSError, ValueError) as e:
+                self.log.info("Skipping file", file=str(file_path), error=str(e))
+                continue
+
+        return working_copy

kodit/domain/value_objects.py

@@ -1,17 +1,129 @@
-"""Domain value objects and DTOs."""
+"""Pure domain value objects and DTOs."""
 
 import json
 from dataclasses import dataclass
 from datetime import datetime
-from enum import Enum
+from enum import Enum, IntEnum
 from pathlib import Path
-from typing import Any, ClassVar
+from typing import ClassVar
 
-from sqlalchemy import JSON, DateTime, Integer, Text
-from sqlalchemy.orm import Mapped, mapped_column
+from pydantic import BaseModel
 
-from kodit.domain.entities import Author, Base, File, Snippet, Source
-from kodit.domain.enums import SnippetExtractionStrategy
+
+class SourceType(IntEnum):
+    """The type of source."""
+
+    UNKNOWN = 0
+    FOLDER = 1
+    GIT = 2
+
+
+class SnippetContentType(IntEnum):
+    """Type of snippet content."""
+
+    UNKNOWN = 0
+    ORIGINAL = 1
+    SUMMARY = 2
+
+
+class SnippetContent(BaseModel):
+    """Snippet content domain value object."""
+
+    type: SnippetContentType
+    value: str
+    language: str
+
+
+class SnippetSearchResult(BaseModel):
+    """Domain result object for snippet searches."""
+
+    snippet_id: int
+    content: str
+    summary: str
+    score: float
+    file_path: Path
+    language: str | None = None
+    authors: list[str] = []
+
+
+@dataclass(frozen=True)
+class LanguageExtensions:
+    """Value object for language to file extension mappings."""
+
+    language: str
+    extensions: list[str]
+
+    @classmethod
+    def get_supported_languages(cls) -> list[str]:
+        """Get all supported programming languages."""
+        return [
+            "python",
+            "javascript",
+            "typescript",
+            "java",
+            "c",
+            "cpp",
+            "csharp",
+            "go",
+            "rust",
+            "php",
+            "ruby",
+            "swift",
+            "kotlin",
+            "scala",
+            "r",
+            "sql",
+            "html",
+            "css",
+            "json",
+            "yaml",
+            "xml",
+            "markdown",
+            "shell",
+        ]
+
+    @classmethod
+    def get_extensions_for_language(cls, language: str) -> list[str]:
+        """Get file extensions for a given language."""
+        language_map = {
+            "python": [".py", ".pyw", ".pyi"],
+            "javascript": [".js", ".jsx", ".mjs"],
+            "typescript": [".ts", ".tsx"],
+            "java": [".java"],
+            "c": [".c", ".h"],
+            "cpp": [".cpp", ".cc", ".cxx", ".hpp", ".hxx"],
+            "csharp": [".cs"],
+            "go": [".go"],
+            "rust": [".rs"],
+            "php": [".php"],
+            "ruby": [".rb"],
+            "swift": [".swift"],
+            "kotlin": [".kt", ".kts"],
+            "scala": [".scala", ".sc"],
+            "r": [".r", ".R"],
+            "sql": [".sql"],
+            "html": [".html", ".htm"],
+            "css": [".css", ".scss", ".sass", ".less"],
+            "json": [".json"],
+            "yaml": [".yaml", ".yml"],
+            "xml": [".xml"],
+            "markdown": [".md", ".markdown"],
+            "shell": [".sh", ".bash", ".zsh", ".fish"],
+        }
+        return language_map.get(language.lower(), [])
+
+    @classmethod
+    def is_supported_language(cls, language: str) -> bool:
+        """Check if a language is supported."""
+        return language.lower() in cls.get_supported_languages()
+
+    @classmethod
+    def get_extensions_or_fallback(cls, language: str) -> list[str]:
+        """Get extensions for language or return language as extension if not found."""
+        language_lower = language.lower()
+        if cls.is_supported_language(language_lower):
+            return cls.get_extensions_for_language(language_lower)
+        return [language_lower]
 
 
 class SearchType(Enum):
@@ -22,14 +134,6 @@ class SearchType(Enum):
     HYBRID = "hybrid"
 
 
-@dataclass
-class SnippetExtractionRequest:
-    """Domain model for snippet extraction request."""
-
-    file_path: Path
-    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED
-
-
 @dataclass
 class SnippetExtractionResult:
     """Domain model for snippet extraction result."""
@@ -101,6 +205,7 @@ class SnippetSearchFilters:
     created_after: datetime | None = None
     created_before: datetime | None = None
     source_repo: str | None = None
+    file_path: str | None = None
 
     @classmethod
     def from_cli_params(
@@ -357,16 +462,6 @@ class IndexView:
     source: str | None = None
 
 
-@dataclass
-class SnippetWithContext:
-    """Domain model for snippet with associated context information."""
-
-    source: Source
-    file: File
-    authors: list[Author]
-    snippet: Snippet
-
-
 class LanguageMapping:
     """Value object for language-to-extension mappings.
 
@@ -536,38 +631,33 @@ class LanguageMapping:
         return [language_lower]
 
 
-# Database models for value objects
-class BM25DocumentModel(Base):
-    """BM25 document model."""
-
-    __tablename__ = "bm25_documents"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    content: Mapped[str] = mapped_column(Text, nullable=False)
-    document_metadata: Mapped[dict[str, Any] | None] = mapped_column(
-        JSON, nullable=True
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False
-    )
-
-
-class VectorDocumentModel(Base):
-    """Vector document model."""
-
-    __tablename__ = "vector_documents"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    content: Mapped[str] = mapped_column(Text, nullable=False)
-    document_metadata: Mapped[dict[str, Any] | None] = mapped_column(
-        JSON, nullable=True
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False
-    )
+class SnippetQuery(BaseModel):
+    """Domain query object for snippet searches."""
+
+    text: str
+    search_type: SearchType = SearchType.HYBRID
+    filters: SnippetSearchFilters = SnippetSearchFilters()
+    top_k: int = 10
+
+
+class SnippetExtractionStrategy(str, Enum):
+    """Different strategies for extracting snippets from files."""
+
+    METHOD_BASED = "method_based"
+
+
+@dataclass
+class SnippetExtractionRequest:
+    """Domain model for snippet extraction request."""
+
+    file_path: Path
+    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED
+
+
+class FileProcessingStatus(IntEnum):
+    """File processing status."""
+
+    CLEAN = 0
+    ADDED = 1
+    MODIFIED = 2
+    DELETED = 3

kodit/infrastructure/cloning/git/working_copy.py

@@ -6,7 +6,7 @@ from pathlib import Path
 import git
 import structlog
 
-from kodit.infrastructure.git.git_utils import sanitize_git_url
+from kodit.domain.entities import WorkingCopy
 
 
 class GitWorkingCopyProvider:
@@ -17,15 +17,17 @@ class GitWorkingCopyProvider:
         self.clone_dir = clone_dir
         self.log = structlog.get_logger(__name__)
 
+    def get_clone_path(self, uri: str) -> Path:
+        """Get the clone path for a Git working copy."""
+        sanitized_uri = WorkingCopy.sanitize_git_url(uri)
+        dir_hash = hashlib.sha256(str(sanitized_uri).encode("utf-8")).hexdigest()[:16]
+        dir_name = f"repo-{dir_hash}"
+        return self.clone_dir / dir_name
+
     async def prepare(self, uri: str) -> Path:
         """Prepare a Git working copy."""
-        # Sanitize the URI for directory name to prevent credential leaks
-        sanitized_uri = sanitize_git_url(uri)
-
-        # Use a repeatable, short sha256 hash of the sanitized URI for the directory
-        dir_hash = hashlib.sha256(sanitized_uri.encode("utf-8")).hexdigest()[:16]
-        dir_name = f"repo-{dir_hash}"
-        clone_path = self.clone_dir / dir_name
+        sanitized_uri = WorkingCopy.sanitize_git_url(uri)
+        clone_path = self.get_clone_path(uri)
         clone_path.mkdir(parents=True, exist_ok=True)
 
         try:
@@ -41,3 +43,10 @@ class GitWorkingCopyProvider:
             self.log.info("Repository already exists, reusing...", uri=sanitized_uri)
 
         return clone_path
+
+    async def sync(self, uri: str) -> Path:
+        """Refresh a Git working copy."""
+        clone_path = self.get_clone_path(uri)
+        repo = git.Repo(clone_path)
+        repo.remotes.origin.pull()
+        return clone_path