kodit 0.2.8__py3-none-any.whl → 0.3.0__py3-none-any.whl

kodit/domain/services/embedding_service.py

@@ -7,10 +7,10 @@ from kodit.domain.entities import EmbeddingType
 from kodit.domain.value_objects import (
     EmbeddingRequest,
     EmbeddingResponse,
+    IndexRequest,
     IndexResult,
-    VectorIndexRequest,
-    VectorSearchQueryRequest,
-    VectorSearchResult,
+    SearchRequest,
+    SearchResult,
 )
 
 
@@ -29,14 +29,12 @@ class VectorSearchRepository(ABC):
 
     @abstractmethod
     def index_documents(
-        self, request: VectorIndexRequest
+        self, request: IndexRequest
     ) -> AsyncGenerator[list[IndexResult], None]:
         """Index documents for vector search."""
 
     @abstractmethod
-    async def search(
-        self, request: VectorSearchQueryRequest
-    ) -> Sequence[VectorSearchResult]:
+    async def search(self, request: SearchRequest) -> Sequence[SearchResult]:
         """Search documents using vector similarity."""
 
     @abstractmethod
@@ -65,7 +63,7 @@ class EmbeddingDomainService:
         self.vector_search_repository = vector_search_repository
 
     async def index_documents(
-        self, request: VectorIndexRequest
+        self, request: IndexRequest
     ) -> AsyncGenerator[list[IndexResult], None]:
         """Index documents using domain business rules.
 
@@ -94,15 +92,13 @@ class EmbeddingDomainService:
             return
 
         # Domain logic: create new request with validated documents
-        validated_request = VectorIndexRequest(documents=valid_documents)
+        validated_request = IndexRequest(documents=valid_documents)
         async for result in self.vector_search_repository.index_documents(
             validated_request
         ):
             yield result
 
-    async def search(
-        self, request: VectorSearchQueryRequest
-    ) -> Sequence[VectorSearchResult]:
+    async def search(self, request: SearchRequest) -> Sequence[SearchResult]:
         """Search documents using domain business rules.
 
         Args:
@@ -124,8 +120,8 @@ class EmbeddingDomainService:
 
         # Domain logic: normalize query
         normalized_query = request.query.strip()
-        normalized_request = VectorSearchQueryRequest(
-            query=normalized_query, top_k=request.top_k
+        normalized_request = SearchRequest(
+            query=normalized_query, top_k=request.top_k, snippet_ids=request.snippet_ids
         )
 
         return await self.vector_search_repository.search(normalized_request)

kodit/domain/services/snippet_service.py

@@ -0,0 +1,198 @@
+"""Domain service for snippet operations."""
+
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from kodit.domain.entities import Snippet
+from kodit.domain.enums import SnippetExtractionStrategy
+from kodit.domain.interfaces import ProgressCallback
+from kodit.domain.repositories import FileRepository, SnippetRepository
+from kodit.domain.services.snippet_extraction_service import (
+    SnippetExtractionDomainService,
+)
+from kodit.domain.value_objects import (
+    MultiSearchRequest,
+    SnippetExtractionRequest,
+    SnippetListItem,
+)
+from kodit.reporting import Reporter
+
+
+class SnippetDomainService:
+    """Domain service for snippet-related operations.
+
+    This service consolidates snippet operations that were previously
+    spread between application services. It handles:
+    - Snippet extraction from files
+    - Snippet persistence
+    - Snippet querying and filtering
+    """
+
+    def __init__(
+        self,
+        snippet_extraction_service: SnippetExtractionDomainService,
+        snippet_repository: SnippetRepository,
+        file_repository: FileRepository,
+    ) -> None:
+        """Initialize the snippet domain service.
+
+        Args:
+            snippet_extraction_service: Service for extracting snippets from files
+            snippet_repository: Repository for snippet persistence
+            file_repository: Repository for file operations
+
+        """
+        self.snippet_extraction_service = snippet_extraction_service
+        self.snippet_repository = snippet_repository
+        self.file_repository = file_repository
+        self.log = structlog.get_logger(__name__)
+
+    async def extract_and_create_snippets(
+        self,
+        index_id: int,
+        strategy: SnippetExtractionStrategy,
+        progress_callback: ProgressCallback | None = None,
+    ) -> list[Snippet]:
+        """Extract snippets from all files in an index and persist them.
+
+        This method combines the extraction and persistence logic that was
+        previously split between domain and application services.
+
+        Args:
+            index_id: The ID of the index to create snippets for
+            strategy: The extraction strategy to use
+            progress_callback: Optional callback for progress reporting
+
+        Returns:
+            List of created Snippet entities with IDs assigned
+
+        """
+        files = await self.file_repository.get_files_for_index(index_id)
+        created_snippets = []
+
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "create_snippets", len(files), "Creating snippets from files..."
+        )
+
+        for i, file in enumerate(files, 1):
+            if not self._should_process_file(file):
+                continue
+
+            try:
+                # Extract snippets from file
+                request = SnippetExtractionRequest(Path(file.cloned_path), strategy)
+                result = await self.snippet_extraction_service.extract_snippets(request)
+
+                # Create and persist snippet entities
+                for snippet_content in result.snippets:
+                    snippet = Snippet(
+                        file_id=file.id,
+                        index_id=index_id,
+                        content=snippet_content,
+                    )
+                    saved_snippet = await self.snippet_repository.save(snippet)
+                    created_snippets.append(saved_snippet)
+
+            except (OSError, ValueError) as e:
+                self.log.debug(
+                    "Skipping file",
+                    file=file.cloned_path,
+                    error=str(e),
+                )
+                continue
+
+            await reporter.step(
+                "create_snippets",
+                current=i,
+                total=len(files),
+                message=f"Processing {file.cloned_path}...",
+            )
+
+        await reporter.done("create_snippets")
+        return created_snippets
+
+    async def get_snippets_for_index(self, index_id: int) -> list[Snippet]:
+        """Get all snippets for a specific index.
+
+        Args:
+            index_id: The ID of the index
+
+        Returns:
+            List of Snippet entities for the index
+
+        """
+        # This delegates to the repository but provides a domain-level interface
+        return list(await self.snippet_repository.get_by_index(index_id))
+
+    async def update_snippet_content(self, snippet_id: int, content: str) -> None:
+        """Update the content of an existing snippet.
+
+        Args:
+            snippet_id: The ID of the snippet to update
+            content: The new content for the snippet
+
+        """
+        # Get the snippet first to ensure it exists
+        snippet = await self.snippet_repository.get(snippet_id)
+        if not snippet:
+            msg = f"Snippet not found: {snippet_id}"
+            raise ValueError(msg)
+
+        # Update the content
+        snippet.content = content
+        await self.snippet_repository.save(snippet)
+
+    async def delete_snippets_for_index(self, index_id: int) -> None:
+        """Delete all snippets for a specific index.
+
+        Args:
+            index_id: The ID of the index
+
+        """
+        await self.snippet_repository.delete_by_index(index_id)
+
+    async def search_snippets(
+        self, request: MultiSearchRequest
+    ) -> list[SnippetListItem]:
+        """Search snippets with filters.
+
+        Args:
+            request: The search request containing filters
+
+        Returns:
+            List of matching snippet items
+
+        """
+        return list(await self.snippet_repository.search(request))
+
+    async def list_snippets(
+        self, file_path: str | None = None, source_uri: str | None = None
+    ) -> list[SnippetListItem]:
+        """List snippets with optional filtering.
+
+        Args:
+            file_path: Optional file path to filter by
+            source_uri: Optional source URI to filter by
+
+        Returns:
+            List of snippet items matching the criteria
+
+        """
+        return list(await self.snippet_repository.list_snippets(file_path, source_uri))
+
+    def _should_process_file(self, file: Any) -> bool:
+        """Check if a file should be processed for snippet extraction.
+
+        Args:
+            file: The file to check
+
+        Returns:
+            True if the file should be processed
+
+        """
+        # Skip unsupported file types
+        mime_blacklist = ["unknown/unknown"]
+        return file.mime_type not in mime_blacklist

kodit/domain/value_objects.py

@@ -4,7 +4,12 @@ from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
 from pathlib import Path
+from typing import Any, ClassVar
 
+from sqlalchemy import JSON, DateTime, Integer, Text
+from sqlalchemy.orm import Mapped, mapped_column
+
+from kodit.domain.entities import Base
 from kodit.domain.enums import SnippetExtractionStrategy
 
 
@@ -40,6 +45,14 @@ class Document:
     text: str
 
 
+@dataclass
+class DocumentSearchResult:
+    """Generic document search result model."""
+
+    snippet_id: int
+    score: float
+
+
 @dataclass
 class SearchResult:
     """Generic search result model."""
@@ -56,12 +69,12 @@ class IndexRequest:
 
 
 @dataclass
-class SimpleSearchRequest:
+class SearchRequest:
     """Generic search request (single query string)."""
 
     query: str
     top_k: int = 10
-    search_type: SearchType = SearchType.BM25
+    snippet_ids: list[int] | None = None
 
 
 @dataclass
@@ -78,17 +91,75 @@ class IndexResult:
     snippet_id: int
 
 
-# Legacy aliases for backward compatibility
-BM25Document = Document
-BM25SearchResult = SearchResult
-BM25IndexRequest = IndexRequest
-BM25SearchRequest = SimpleSearchRequest
-BM25DeleteRequest = DeleteRequest
-
-VectorSearchRequest = Document
-VectorSearchResult = SearchResult
-VectorIndexRequest = IndexRequest
-VectorSearchQueryRequest = SimpleSearchRequest
+@dataclass(frozen=True)
+class SnippetSearchFilters:
+    """Value object for filtering snippet search results."""
+
+    language: str | None = None
+    author: str | None = None
+    created_after: datetime | None = None
+    created_before: datetime | None = None
+    source_repo: str | None = None
+
+    @classmethod
+    def from_cli_params(
+        cls,
+        language: str | None = None,
+        author: str | None = None,
+        created_after: str | None = None,
+        created_before: str | None = None,
+        source_repo: str | None = None,
+    ) -> "SnippetSearchFilters | None":
+        """Create SnippetSearchFilters from CLI parameters.
+
+        Args:
+            language: Programming language filter (e.g., python, go, javascript)
+            author: Author name filter
+            created_after: Date string in YYYY-MM-DD format for filtering snippets
+            created after
+            created_before: Date string in YYYY-MM-DD format for filtering snippets
+            created before
+            source_repo: Source repository filter (e.g., github.com/example/repo)
+
+        Returns:
+            SnippetSearchFilters instance if any filters are provided, None otherwise
+
+        Raises:
+            ValueError: If date strings are in invalid format
+
+        """
+        # Only create filters if at least one parameter is provided
+        if not any([language, author, created_after, created_before, source_repo]):
+            return None
+
+        # Parse date strings if provided
+        parsed_created_after = None
+        if created_after:
+            try:
+                parsed_created_after = datetime.fromisoformat(created_after)
+            except ValueError as e:
+                raise ValueError(
+                    f"Invalid date format for created_after: {created_after}. "
+                    "Expected ISO 8601 format (YYYY-MM-DD)"
+                ) from e
+
+        parsed_created_before = None
+        if created_before:
+            try:
+                parsed_created_before = datetime.fromisoformat(created_before)
+            except ValueError as e:
+                raise ValueError(
+                    f"Invalid date format for created_before: {created_before}. "
+                    "Expected ISO 8601 format (YYYY-MM-DD)"
+                ) from e
+
+        return cls(
+            language=language,
+            author=author,
+            created_after=parsed_created_after,
+            created_before=parsed_created_before,
+            source_repo=source_repo,
+        )
 
 
 @dataclass
@@ -99,6 +170,7 @@ class MultiSearchRequest:
     text_query: str | None = None
     code_query: str | None = None
     keywords: list[str] | None = None
+    filters: SnippetSearchFilters | None = None
 
 
 @dataclass
@@ -196,14 +268,6 @@ class EnrichmentIndexRequest:
     requests: list[EnrichmentRequest]
 
 
-@dataclass
-class EnrichmentSearchRequest:
-    """Domain model for enrichment search request."""
-
-    query: str
-    top_k: int = 10
-
-
 @dataclass
 class IndexView:
     """Domain model for index information."""
@@ -213,3 +277,219 @@ class IndexView:
     num_snippets: int
     updated_at: datetime | None = None
     source: str | None = None
+
+
+@dataclass
+class SnippetListItem:
+    """Domain model for snippet list item with file information."""
+
+    id: int
+    file_path: str
+    content: str
+    source_uri: str
+
+
+class LanguageMapping:
+    """Value object for language-to-extension mappings.
+
+    This encapsulates the domain knowledge of programming languages and their
+    associated file extensions. It provides bidirectional mapping capabilities
+    and is designed to be immutable and reusable across the application.
+    """
+
+    # Comprehensive mapping of language names to their file extensions
+    _LANGUAGE_TO_EXTENSIONS: ClassVar[dict[str, list[str]]] = {
+        "python": ["py", "pyw", "pyx", "pxd"],
+        "go": ["go"],
+        "javascript": ["js", "jsx", "mjs"],
+        "typescript": ["ts", "tsx"],
+        "java": ["java"],
+        "csharp": ["cs"],
+        "cpp": ["cpp", "cc", "cxx", "hpp"],
+        "c": ["c", "h"],
+        "rust": ["rs"],
+        "php": ["php"],
+        "ruby": ["rb"],
+        "swift": ["swift"],
+        "kotlin": ["kt", "kts"],
+        "scala": ["scala"],
+        "r": ["r", "R"],
+        "matlab": ["m"],
+        "perl": ["pl", "pm"],
+        "bash": ["sh", "bash"],
+        "powershell": ["ps1"],
+        "sql": ["sql"],
+        "html": ["html", "htm"],
+        "css": ["css", "scss", "sass"],
+        "yaml": ["yml", "yaml"],
+        "json": ["json"],
+        "xml": ["xml"],
+        "markdown": ["md", "markdown"],
+    }
+
+    @classmethod
+    def get_extensions_for_language(cls, language: str) -> list[str]:
+        """Get file extensions for a given language.
+
+        Args:
+            language: The programming language name (case-insensitive)
+
+        Returns:
+            List of file extensions (without leading dots) for the language
+
+        Raises:
+            ValueError: If the language is not supported
+
+        """
+        language_lower = language.lower()
+        extensions = cls._LANGUAGE_TO_EXTENSIONS.get(language_lower)
+
+        if extensions is None:
+            raise ValueError(f"Unsupported language: {language}")
+
+        return extensions.copy()  # Return a copy to prevent modification
+
+    @classmethod
+    def get_language_for_extension(cls, extension: str) -> str:
+        """Get language for a given file extension.
+
+        Args:
+            extension: The file extension (with or without leading dot)
+
+        Returns:
+            The programming language name
+
+        Raises:
+            ValueError: If the extension is not supported
+
+        """
+        # Remove leading dot if present
+        ext_clean = extension.removeprefix(".").lower()
+
+        # Search through all languages to find matching extension
+        for language, extensions in cls._LANGUAGE_TO_EXTENSIONS.items():
+            if ext_clean in extensions:
+                return language
+
+        raise ValueError(f"Unsupported file extension: {extension}")
+
+    @classmethod
+    def get_extension_to_language_map(cls) -> dict[str, str]:
+        """Get a mapping from file extensions to language names.
+
+        Returns:
+            Dictionary mapping file extensions (without leading dots) to language names
+
+        """
+        extension_map = {}
+        for language, extensions in cls._LANGUAGE_TO_EXTENSIONS.items():
+            for extension in extensions:
+                extension_map[extension] = language
+        return extension_map
+
+    @classmethod
+    def get_supported_languages(cls) -> list[str]:
+        """Get list of all supported programming languages.
+
+        Returns:
+            List of supported language names
+
+        """
+        return list(cls._LANGUAGE_TO_EXTENSIONS.keys())
+
+    @classmethod
+    def get_supported_extensions(cls) -> list[str]:
+        """Get list of all supported file extensions.
+
+        Returns:
+            List of supported file extensions (without leading dots)
+
+        """
+        extensions = []
+        for ext_list in cls._LANGUAGE_TO_EXTENSIONS.values():
+            extensions.extend(ext_list)
+        return extensions
+
+    @classmethod
+    def is_supported_language(cls, language: str) -> bool:
+        """Check if a language is supported.
+
+        Args:
+            language: The programming language name (case-insensitive)
+
+        Returns:
+            True if the language is supported, False otherwise
+
+        """
+        return language.lower() in cls._LANGUAGE_TO_EXTENSIONS
+
+    @classmethod
+    def is_supported_extension(cls, extension: str) -> bool:
+        """Check if a file extension is supported.
+
+        Args:
+            extension: The file extension (with or without leading dot)
+
+        Returns:
+            True if the extension is supported, False otherwise
+
+        """
+        try:
+            cls.get_language_for_extension(extension)
+        except ValueError:
+            return False
+        return True
+
+    @classmethod
+    def get_extensions_with_fallback(cls, language: str) -> list[str]:
+        """Get file extensions for a language, falling back to passed language name.
+
+        Args:
+            language: The programming language name (case-insensitive)
+
+        Returns:
+            List of file extensions (without leading dots) for the language, or
+            [language.lower()] if not found.
+
+        """
+        language_lower = language.lower()
+        if cls.is_supported_language(language_lower):
+            return cls.get_extensions_for_language(language_lower)
+        return [language_lower]
+
+
+# Database models for value objects
+class BM25DocumentModel(Base):
+    """BM25 document model."""
+
+    __tablename__ = "bm25_documents"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    content: Mapped[str] = mapped_column(Text, nullable=False)
+    document_metadata: Mapped[dict[str, Any] | None] = mapped_column(
+        JSON, nullable=True
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False
+    )
+
+
+class VectorDocumentModel(Base):
+    """Vector document model."""
+
+    __tablename__ = "vector_documents"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    content: Mapped[str] = mapped_column(Text, nullable=False)
+    document_metadata: Mapped[dict[str, Any] | None] = mapped_column(
+        JSON, nullable=True
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False
+    )

kodit/infrastructure/bm25/local_bm25_repository.py

@@ -12,10 +12,10 @@ import structlog
 
 from kodit.domain.services.bm25_service import BM25Repository
 from kodit.domain.value_objects import (
-    BM25DeleteRequest,
-    BM25IndexRequest,
-    BM25SearchRequest,
-    BM25SearchResult,
+    DeleteRequest,
+    IndexRequest,
+    SearchRequest,
+    SearchResult,
 )
 
 if TYPE_CHECKING:
@@ -68,7 +68,7 @@ class LocalBM25Repository(BM25Repository):
             show_progress=True,
         )
 
-    async def index_documents(self, request: BM25IndexRequest) -> None:
+    async def index_documents(self, request: IndexRequest) -> None:
         """Index documents for BM25 search."""
         self.log.debug("Indexing corpus")
         if not request.documents:
@@ -84,7 +84,7 @@ class LocalBM25Repository(BM25Repository):
         async with aiofiles.open(self.index_path / SNIPPET_IDS_FILE, "w") as f:
             await f.write(json.dumps(self.snippet_ids))
 
-    async def search(self, request: BM25SearchRequest) -> list[BM25SearchResult]:
+    async def search(self, request: SearchRequest) -> list[SearchResult]:
         """Search documents using BM25."""
         if request.top_k == 0:
             self.log.warning("Top k is 0, returning empty list")
@@ -117,13 +117,21 @@ class LocalBM25Repository(BM25Repository):
             k=top_k,
         )
         self.log.debug("Raw results", results=results, scores=scores)
-        return [
-            BM25SearchResult(snippet_id=int(result), score=float(score))
-            for result, score in zip(results[0], scores[0], strict=False)
-            if score > 0.0
-        ]
 
-    async def delete_documents(self, request: BM25DeleteRequest) -> None:
+        # Filter results by snippet_ids if provided
+        filtered_results = []
+        for result, score in zip(results[0], scores[0], strict=False):
+            snippet_id = int(result)
+            if score > 0.0 and (
+                request.snippet_ids is None or snippet_id in request.snippet_ids
+            ):
+                filtered_results.append(
+                    SearchResult(snippet_id=snippet_id, score=float(score))
+                )
+
+        return filtered_results
+
+    async def delete_documents(self, request: DeleteRequest) -> None:
         """Delete documents from the index."""
         # request parameter is unused as deletion is not supported
         # ruff: noqa: ARG002