kodit 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.3'
-__version_tuple__ = version_tuple = (0, 2, 3)
+__version__ = version = '0.2.5'
+__version_tuple__ = version_tuple = (0, 2, 5)

kodit/application/__init__.py

@@ -0,0 +1 @@
+"""Application layer for Kodit."""

kodit/application/commands/__init__.py

@@ -0,0 +1 @@
+"""Application commands for Kodit."""

kodit/application/commands/snippet_commands.py

@@ -0,0 +1,22 @@
+"""Application commands for snippet operations."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from kodit.domain.enums import SnippetExtractionStrategy
+
+
+@dataclass
+class ExtractSnippetsCommand:
+    """Application command for extracting snippets from files."""
+
+    file_path: Path
+    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED
+
+
+@dataclass
+class CreateIndexSnippetsCommand:
+    """Application command for creating snippets for an entire index."""
+
+    index_id: int
+    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED

kodit/application/services/__init__.py

@@ -0,0 +1 @@
+"""Application services for Kodit."""

kodit/application/services/indexing_application_service.py

@@ -0,0 +1,363 @@
+"""Application service for indexing operations."""
+
+import structlog
+
+from kodit.application.commands.snippet_commands import CreateIndexSnippetsCommand
+from kodit.application.services.snippet_application_service import (
+    SnippetApplicationService,
+)
+from kodit.domain.entities import Snippet
+from kodit.domain.enums import SnippetExtractionStrategy
+from kodit.domain.interfaces import ProgressCallback
+from kodit.domain.services.bm25_service import BM25DomainService
+from kodit.domain.services.embedding_service import EmbeddingDomainService
+from kodit.domain.services.enrichment_service import EnrichmentDomainService
+from kodit.domain.services.indexing_service import IndexingDomainService
+from kodit.domain.services.source_service import SourceService
+from kodit.domain.value_objects import (
+    BM25Document,
+    BM25IndexRequest,
+    BM25SearchRequest,
+    BM25SearchResult,
+    EnrichmentIndexRequest,
+    EnrichmentRequest,
+    FusionRequest,
+    IndexCreateRequest,
+    IndexView,
+    MultiSearchRequest,
+    MultiSearchResult,
+    VectorIndexRequest,
+    VectorSearchQueryRequest,
+    VectorSearchRequest,
+)
+from kodit.log import log_event
+from kodit.reporting import Reporter
+
+
+class IndexingApplicationService:
+    """Application service for indexing operations.
+
+    This service orchestrates the business logic for creating, listing, and running
+    code indexes. It coordinates between domain services and provides a clean API
+    for index management.
+    """
+
+    def __init__(  # noqa: PLR0913
+        self,
+        indexing_domain_service: IndexingDomainService,
+        source_service: SourceService,
+        bm25_service: BM25DomainService,
+        code_search_service: EmbeddingDomainService,
+        text_search_service: EmbeddingDomainService,
+        enrichment_service: EnrichmentDomainService,
+        snippet_application_service: SnippetApplicationService,
+    ) -> None:
+        """Initialize the indexing application service.
+
+        Args:
+            indexing_domain_service: The indexing domain service.
+            source_service: The source service for source validation.
+            bm25_service: The BM25 domain service for keyword search.
+            code_search_service: The code search domain service.
+            text_search_service: The text search domain service.
+            enrichment_service: The enrichment domain service.
+            snippet_application_service: The snippet application service.
+
+        """
+        self.indexing_domain_service = indexing_domain_service
+        self.source_service = source_service
+        self.snippet_application_service = snippet_application_service
+        self.log = structlog.get_logger(__name__)
+        self.bm25_service = bm25_service
+        self.code_search_service = code_search_service
+        self.text_search_service = text_search_service
+        self.enrichment_service = enrichment_service
+
+    async def create_index(self, source_id: int) -> IndexView:
+        """Create a new index for a source.
+
+        Args:
+            source_id: The ID of the source to create an index for.
+
+        Returns:
+            An IndexView representing the newly created index.
+
+        Raises:
+            ValueError: If the source doesn't exist.
+
+        """
+        log_event("kodit.index.create")
+
+        # Check if the source exists
+        source = await self.source_service.get(source_id)
+
+        # Create the index
+        request = IndexCreateRequest(source_id=source.id)
+        return await self.indexing_domain_service.create_index(request)
+
+    async def list_indexes(self) -> list[IndexView]:
+        """List all available indexes with their details.
+
+        Returns:
+            A list of IndexView objects containing information about each index.
+
+        """
+        indexes = await self.indexing_domain_service.list_indexes()
+
+        # Help Kodit by measuring how much people are using indexes
+        log_event(
+            "kodit.index.list",
+            {
+                "num_indexes": len(indexes),
+                "num_snippets": sum([index.num_snippets for index in indexes]),
+            },
+        )
+
+        return indexes
+
+    async def run_index(
+        self, index_id: int, progress_callback: ProgressCallback | None = None
+    ) -> None:
+        """Run the indexing process for a specific index.
+
+        Args:
+            index_id: The ID of the index to run.
+            progress_callback: Optional progress callback for reporting progress.
+
+        Raises:
+            ValueError: If the index doesn't exist.
+
+        """
+        log_event("kodit.index.run")
+
+        # Get and validate index
+        index = await self.indexing_domain_service.get_index(index_id)
+        if not index:
+            msg = f"Index not found: {index_id}"
+            raise ValueError(msg)
+
+        # Delete old snippets so we don't duplicate
+        await self.indexing_domain_service.delete_all_snippets(index.id)
+
+        # Create snippets for supported file types using the snippet application service
+        self.log.info("Creating snippets for files", index_id=index.id)
+        command = CreateIndexSnippetsCommand(
+            index_id=index.id, strategy=SnippetExtractionStrategy.METHOD_BASED
+        )
+        await self.snippet_application_service.create_snippets_for_index(
+            command, progress_callback
+        )
+
+        snippets = await self.indexing_domain_service.get_snippets_for_index(index.id)
+
+        # Create BM25 index
+        self.log.info("Creating keyword index")
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start("bm25_index", len(snippets), "Creating keyword index...")
+        await self._create_bm25_index(snippets, progress_callback)
+        await reporter.done("bm25_index", "Keyword index created")
+
+        # Create code embeddings
+        self.log.info("Creating semantic code index")
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "code_embeddings", len(snippets), "Creating code embeddings..."
+        )
+        await self._create_code_embeddings(snippets, progress_callback)
+        await reporter.done("code_embeddings")
+
+        # Enrich snippets
+        self.log.info("Enriching snippets", num_snippets=len(snippets))
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start("enrichment", len(snippets), "Enriching snippets...")
+        await self._enrich_snippets(snippets, progress_callback)
+        await reporter.done("enrichment")
+
+        # Create text embeddings
+        self.log.info("Creating semantic text index")
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "text_embeddings", len(snippets), "Creating text embeddings..."
+        )
+        await self._create_text_embeddings(snippets, progress_callback)
+        await reporter.done("text_embeddings")
+
+        # Update index timestamp
+        await self.indexing_domain_service.update_index_timestamp(index.id)
+
+    async def _create_bm25_index(
+        self, snippets: list[Snippet], progress_callback: ProgressCallback | None = None
+    ) -> None:
+        """Create BM25 keyword index."""
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start("bm25_index", len(snippets), "Creating keyword index...")
+        await self.bm25_service.index_documents(
+            BM25IndexRequest(
+                documents=[
+                    BM25Document(snippet_id=snippet.id, text=snippet.content)
+                    for snippet in snippets
+                ]
+            )
+        )
+        await reporter.done("bm25_index", "Keyword index created")
+
+    async def _create_code_embeddings(
+        self, snippets: list[Snippet], progress_callback: ProgressCallback | None = None
+    ) -> None:
+        """Create code embeddings."""
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "code_embeddings", len(snippets), "Creating code embeddings..."
+        )
+        processed = 0
+        async for result in self.code_search_service.index_documents(
+            VectorIndexRequest(
+                documents=[
+                    VectorSearchRequest(snippet.id, snippet.content)
+                    for snippet in snippets
+                ]
+            )
+        ):
+            processed += len(result)
+            await reporter.step(
+                "code_embeddings",
+                processed,
+                len(snippets),
+                "Creating code embeddings...",
+            )
+        await reporter.done("code_embeddings")
+
+    async def _enrich_snippets(
+        self, snippets: list[Snippet], progress_callback: ProgressCallback | None = None
+    ) -> None:
+        """Enrich snippets with additional context."""
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start("enrichment", len(snippets), "Enriching snippets...")
+        enriched_contents = []
+        enrichment_request = EnrichmentIndexRequest(
+            requests=[
+                EnrichmentRequest(snippet_id=snippet.id, text=snippet.content)
+                for snippet in snippets
+            ]
+        )
+
+        processed = 0
+        async for result in self.enrichment_service.enrich_documents(
+            enrichment_request
+        ):
+            # Find the snippet by ID
+            snippet = next(s for s in snippets if s.id == result.snippet_id)
+            if snippet:
+                # Update the content in the local entity for subsequent processing
+                enriched_content = result.text + "\n\n```\n" + snippet.content + "\n```"
+                snippet.content = enriched_content
+
+                # UPDATE the existing snippet entity instead of creating a new one
+                # This follows DDD principles and avoids duplicates
+                await self.indexing_domain_service.update_snippet_content(
+                    snippet.id, enriched_content
+                )
+                enriched_contents.append(result)
+
+            processed += 1
+            await reporter.step(
+                "enrichment", processed, len(snippets), "Enriching snippets..."
+            )
+
+        await reporter.done("enrichment")
+
+    async def _create_text_embeddings(
+        self, snippets: list[Snippet], progress_callback: ProgressCallback | None = None
+    ) -> None:
+        """Create text embeddings."""
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "text_embeddings", len(snippets), "Creating text embeddings..."
+        )
+        processed = 0
+        async for result in self.text_search_service.index_documents(
+            VectorIndexRequest(
+                documents=[
+                    VectorSearchRequest(snippet.id, snippet.content)
+                    for snippet in snippets
+                ]
+            )
+        ):
+            processed += len(result)
+            await reporter.step(
+                "text_embeddings",
+                processed,
+                len(snippets),
+                "Creating text embeddings...",
+            )
+        await reporter.done("text_embeddings")
+
+    async def search(self, request: MultiSearchRequest) -> list[MultiSearchResult]:
+        """Search for relevant data.
+
+        Args:
+            request: The search request.
+
+        Returns:
+            A list of search results.
+
+        """
+        log_event("kodit.index.search")
+
+        fusion_list: list[list[FusionRequest]] = []
+        if request.keywords:
+            # Gather results for each keyword
+            result_ids: list[BM25SearchResult] = []
+            for keyword in request.keywords:
+                results = await self.bm25_service.search(
+                    BM25SearchRequest(query=keyword, top_k=request.top_k)
+                )
+                result_ids.extend(results)
+
+            fusion_list.append(
+                [FusionRequest(id=x.snippet_id, score=x.score) for x in result_ids]
+            )
+
+        # Compute embedding for semantic query
+        if request.code_query:
+            query_embedding = await self.code_search_service.search(
+                VectorSearchQueryRequest(query=request.code_query, top_k=request.top_k)
+            )
+            fusion_list.append(
+                [FusionRequest(id=x.snippet_id, score=x.score) for x in query_embedding]
+            )
+
+        if request.text_query:
+            query_embedding = await self.text_search_service.search(
+                VectorSearchQueryRequest(query=request.text_query, top_k=request.top_k)
+            )
+            fusion_list.append(
+                [FusionRequest(id=x.snippet_id, score=x.score) for x in query_embedding]
+            )
+
+        if len(fusion_list) == 0:
+            return []
+
+        # Combine all results together with RFF if required
+        final_results = self.indexing_domain_service.perform_fusion(
+            rankings=fusion_list,
+            k=60,
+        )
+
+        # Only keep top_k results
+        final_results = final_results[: request.top_k]
+
+        # Get snippets from database (up to top_k)
+        search_results = await self.indexing_domain_service.get_snippets_by_ids(
+            [x.id for x in final_results]
+        )
+
+        return [
+            MultiSearchResult(
+                id=snippet["id"],
+                uri=file["uri"],
+                content=snippet["content"],
+                original_scores=fr.original_scores,
+            )
+            for (file, snippet), fr in zip(search_results, final_results, strict=True)
+        ]

kodit/application/services/snippet_application_service.py

@@ -0,0 +1,143 @@
+"""Application service for snippet operations."""
+
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from kodit.application.commands.snippet_commands import (
+    CreateIndexSnippetsCommand,
+    ExtractSnippetsCommand,
+)
+from kodit.domain.entities import Snippet
+from kodit.domain.enums import SnippetExtractionStrategy
+from kodit.domain.interfaces import ProgressCallback
+from kodit.domain.repositories import FileRepository, SnippetRepository
+from kodit.domain.services.snippet_extraction_service import (
+    SnippetExtractionDomainService,
+)
+from kodit.domain.value_objects import SnippetExtractionRequest
+from kodit.reporting import Reporter
+
+
+class SnippetApplicationService:
+    """Application service for snippet operations."""
+
+    def __init__(
+        self,
+        snippet_extraction_service: SnippetExtractionDomainService,
+        snippet_repository: SnippetRepository,
+        file_repository: FileRepository,
+    ) -> None:
+        """Initialize the snippet application service.
+
+        Args:
+            snippet_extraction_service: Domain service for snippet extraction
+            snippet_repository: Repository for snippet persistence
+            file_repository: Repository for file operations
+
+        """
+        self.snippet_extraction_service = snippet_extraction_service
+        self.snippet_repository = snippet_repository
+        self.file_repository = file_repository
+        self.log = structlog.get_logger(__name__)
+
+    async def extract_snippets_from_file(
+        self, command: ExtractSnippetsCommand
+    ) -> list[Snippet]:
+        """Application use case: extract snippets from a single file.
+
+        Args:
+            command: The extract snippets command
+
+        Returns:
+            List of extracted snippets
+
+        """
+        request = SnippetExtractionRequest(command.file_path, command.strategy)
+        result = await self.snippet_extraction_service.extract_snippets(request)
+
+        # Convert domain result to persistence model
+        return [
+            Snippet(
+                file_id=0, index_id=0, content=snippet_text
+            )  # IDs will be set later
+            for snippet_text in result.snippets
+        ]
+
+    def _should_process_file(self, file: Any) -> bool:
+        """Check if a file should be processed for snippet extraction.
+
+        Args:
+            file: The file to check
+
+        Returns:
+            True if the file should be processed
+
+        """
+        # Skip unsupported file types
+        mime_blacklist = ["unknown/unknown"]
+        return file.mime_type not in mime_blacklist
+
+    async def _extract_snippets_from_file(
+        self, file: Any, strategy: SnippetExtractionStrategy
+    ) -> list[str]:
+        """Extract snippets from a single file."""
+        command = ExtractSnippetsCommand(
+            file_path=Path(file.cloned_path),
+            strategy=strategy,
+        )
+        snippets = await self.extract_snippets_from_file(command)
+        return [snippet.content for snippet in snippets]
+
+    async def create_snippets_for_index(
+        self,
+        command: CreateIndexSnippetsCommand,
+        progress_callback: ProgressCallback | None = None,
+    ) -> None:
+        """Create snippets for all files in an index.
+
+        Args:
+            command: The create index snippets command
+            progress_callback: Optional progress callback for reporting progress
+
+        """
+        files = await self.file_repository.get_files_for_index(command.index_id)
+
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "create_snippets", len(files), "Creating snippets from files..."
+        )
+
+        for i, file in enumerate(files, 1):
+            try:
+                if not self._should_process_file(file):
+                    continue
+
+                snippet_contents = await self._extract_snippets_from_file(
+                    file, command.strategy
+                )
+                for snippet_content in snippet_contents:
+                    snippet = Snippet(
+                        file_id=file.id,
+                        index_id=command.index_id,
+                        content=snippet_content,
+                    )
+                    await self.snippet_repository.save(snippet)
+
+            except (OSError, ValueError) as e:
+                self.log.debug(
+                    "Skipping file",
+                    file=file.cloned_path,
+                    error=str(e),
+                )
+                continue
+
+            await reporter.step(
+                "create_snippets",
+                current=i,
+                total=len(files),
+                message=f"Processing {file.cloned_path}...",
+            )
+
+        await reporter.done("create_snippets")