kodit 0.3.3__py3-none-any.whl → 0.3.4__py3-none-any.whl

kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.3'
-__version_tuple__ = version_tuple = (0, 3, 3)
+__version__ = version = '0.3.4'
+__version_tuple__ = version_tuple = (0, 3, 4)

kodit/application/factories/code_indexing_factory.py

@@ -13,7 +13,7 @@ from kodit.domain.services.index_query_service import IndexQueryService
 from kodit.domain.services.index_service import (
     IndexDomainService,
 )
-from kodit.domain.value_objects import LanguageMapping, SnippetExtractionStrategy
+from kodit.domain.value_objects import LanguageMapping
 from kodit.infrastructure.bm25.bm25_factory import bm25_repository_factory
 from kodit.infrastructure.embedding.embedding_factory import (
     embedding_domain_service_factory,
@@ -31,15 +31,9 @@ from kodit.infrastructure.enrichment.null_enrichment_provider import (
     NullEnrichmentProvider,
 )
 from kodit.infrastructure.indexing.fusion_service import ReciprocalRankFusionService
-from kodit.infrastructure.snippet_extraction.factories import (
-    create_snippet_query_provider,
-)
-from kodit.infrastructure.snippet_extraction.language_detection_service import (
+from kodit.infrastructure.slicing.language_detection_service import (
     FileSystemLanguageDetectionService,
 )
-from kodit.infrastructure.snippet_extraction.tree_sitter_snippet_extractor import (
-    TreeSitterSnippetExtractor,
-)
 from kodit.infrastructure.sqlalchemy.embedding_repository import (
     SqlAlchemyEmbeddingRepository,
 )
@@ -63,17 +57,9 @@ def create_code_indexing_application_service(
 
     # Create infrastructure services
     language_detector = FileSystemLanguageDetectionService(language_map)
-    query_provider = create_snippet_query_provider()
 
-    # Create snippet extractors
-    method_extractor = TreeSitterSnippetExtractor(query_provider)
-
-    snippet_extractors = {
-        SnippetExtractionStrategy.METHOD_BASED: method_extractor,
-    }
     index_domain_service = IndexDomainService(
         language_detector=language_detector,
-        snippet_extractors=snippet_extractors,
         enrichment_service=enrichment_service,
         clone_dir=app_context.get_clone_dir(),
     )
@@ -136,17 +122,9 @@ def create_fast_test_code_indexing_application_service(
 
     # Create infrastructure services
     language_detector = FileSystemLanguageDetectionService(language_map)
-    query_provider = create_snippet_query_provider()
-
-    # Create snippet extractors
-    method_extractor = TreeSitterSnippetExtractor(query_provider)
 
-    snippet_extractors = {
-        SnippetExtractionStrategy.METHOD_BASED: method_extractor,
-    }
     index_domain_service = IndexDomainService(
         language_detector=language_detector,
-        snippet_extractors=snippet_extractors,
         enrichment_service=enrichment_service,
         clone_dir=app_context.get_clone_dir(),
     )

kodit/application/services/code_indexing_application_service.py

@@ -100,6 +100,11 @@ class CodeIndexingApplicationService:
             self.log.info("No new changes to index", index_id=index.id)
             return
 
+        # Delete the old snippets from the files that have changed
+        await self.index_repository.delete_snippets_by_file_ids(
+            [file.id for file in index.source.working_copy.changed_files() if file.id]
+        )
+
         # Extract and create snippets (domain service handles progress)
         self.log.info("Creating snippets for files", index_id=index.id)
         index = await self.index_domain_service.extract_snippets_from_index(
@@ -115,6 +120,9 @@ class CodeIndexingApplicationService:
             msg = f"Index {index.id} not found after snippet extraction"
             raise ValueError(msg)
         index = flushed_index
+        if len(index.snippets) == 0:
+            self.log.info("No snippets to index after extraction", index_id=index.id)
+            return
 
         # Create BM25 index
         self.log.info("Creating keyword index")
@@ -154,8 +162,8 @@ class CodeIndexingApplicationService:
         # Apply filters if provided
         filtered_snippet_ids: list[int] | None = None
         if request.filters:
-            # Use domain service for filtering
-            prefilter_request = replace(request, top_k=None)
+            # Use domain service for filtering (use large top_k for pre-filtering)
+            prefilter_request = replace(request, top_k=10000)
             snippet_results = await self.index_query_service.search_snippets(
                 prefilter_request
             )

kodit/domain/services/index_service.py

@@ -1,7 +1,6 @@
 """Pure domain service for Index aggregate operations."""
 
 from abc import ABC, abstractmethod
-from collections.abc import Mapping
 from pathlib import Path
 
 import structlog
@@ -13,14 +12,13 @@ from kodit.domain.services.enrichment_service import EnrichmentDomainService
 from kodit.domain.value_objects import (
     EnrichmentIndexRequest,
     EnrichmentRequest,
-    SnippetExtractionRequest,
-    SnippetExtractionResult,
-    SnippetExtractionStrategy,
+    LanguageMapping,
 )
 from kodit.infrastructure.cloning.git.working_copy import GitWorkingCopyProvider
 from kodit.infrastructure.cloning.metadata import FileMetadataExtractor
 from kodit.infrastructure.git.git_utils import is_valid_clone_target
 from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
+from kodit.infrastructure.slicing.slicer import Slicer
 from kodit.reporting import Reporter
 from kodit.utils.path_utils import path_from_uri
 
@@ -33,14 +31,6 @@ class LanguageDetectionService(ABC):
         """Detect the programming language of a file."""
 
 
-class SnippetExtractor(ABC):
-    """Abstract interface for snippet extraction."""
-
-    @abstractmethod
-    async def extract(self, file_path: Path, language: str) -> list[str]:
-        """Extract snippets from a file."""
-
-
 class IndexDomainService:
     """Pure domain service for Index aggregate operations.
 
@@ -54,14 +44,12 @@ class IndexDomainService:
     def __init__(
         self,
         language_detector: LanguageDetectionService,
-        snippet_extractors: Mapping[SnippetExtractionStrategy, SnippetExtractor],
         enrichment_service: EnrichmentDomainService,
         clone_dir: Path,
     ) -> None:
         """Initialize the index domain service."""
         self._clone_dir = clone_dir
         self._language_detector = language_detector
-        self._snippet_extractors = snippet_extractors
         self._enrichment_service = enrichment_service
         self.log = structlog.get_logger(__name__)
 
@@ -99,7 +87,6 @@ class IndexDomainService:
     async def extract_snippets_from_index(
         self,
         index: domain_entities.Index,
-        strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED,
         progress_callback: ProgressCallback | None = None,
     ) -> domain_entities.Index:
         """Extract code snippets from files in the index."""
@@ -109,46 +96,40 @@ class IndexDomainService:
             "Extracting snippets",
             index_id=index.id,
             file_count=file_count,
-            strategy=strategy.value,
         )
 
         # Only create snippets for files that have been added or modified
         files = index.source.working_copy.changed_files()
         index.delete_snippets_for_files(files)
 
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start(
-            "extract_snippets", len(files), "Extracting code snippets..."
-        )
-
-        new_snippets = []
-        for i, domain_file in enumerate(files, 1):
+        # Create a set of languages to extract snippets for
+        extensions = {file.extension() for file in files}
+        languages = []
+        for ext in extensions:
             try:
-                # Extract snippets from file
-                request = SnippetExtractionRequest(
-                    file_path=domain_file.as_path(), strategy=strategy
-                )
-                result = await self._extract_snippets(request)
-                for snippet_text in result.snippets:
-                    snippet = domain_entities.Snippet(
-                        derives_from=[domain_file],
-                    )
-                    snippet.add_original_content(snippet_text, result.language)
-                    new_snippets.append(snippet)
-
-            except (OSError, ValueError) as e:
-                self.log.debug(
-                    "Skipping file for snippet extraction",
-                    file_uri=str(domain_file.uri),
-                    error=str(e),
-                )
+                languages.append(LanguageMapping.get_language_for_extension(ext))
+            except ValueError as e:
+                self.log.info("Skipping", error=str(e))
                 continue
 
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "extract_snippets",
+            len(files) * len(languages),
+            "Extracting code snippets...",
+        )
+        # Calculate snippets for each language
+        slicer = Slicer()
+        for i, language in enumerate(languages):
             await reporter.step(
-                "extract_snippets", i, len(files), f"Processed {domain_file.uri.path}"
+                "extract_snippets",
+                len(files) * (i + 1),
+                len(files) * len(languages),
+                "Extracting code snippets...",
             )
+            s = slicer.extract_snippets(files, language=language)
+            index.snippets.extend(s)
 
-        index.snippets.extend(new_snippets)
         await reporter.done("extract_snippets")
         return index
 
@@ -187,28 +168,6 @@ class IndexDomainService:
         await reporter.done("enrichment")
         return list(snippet_map.values())
 
-    async def _extract_snippets(
-        self, request: SnippetExtractionRequest
-    ) -> SnippetExtractionResult:
-        # Domain logic: validate file exists
-        if not request.file_path.exists():
-            raise ValueError(f"File does not exist: {request.file_path}")
-
-        # Domain logic: detect language
-        language = await self._language_detector.detect_language(request.file_path)
-
-        # Domain logic: choose strategy and extractor
-        if request.strategy not in self._snippet_extractors:
-            raise ValueError(f"Unsupported extraction strategy: {request.strategy}")
-
-        extractor = self._snippet_extractors[request.strategy]
-        snippets = await extractor.extract(request.file_path, language)
-
-        # Domain logic: filter out empty snippets
-        filtered_snippets = [snippet for snippet in snippets if snippet.strip()]
-
-        return SnippetExtractionResult(snippets=filtered_snippets, language=language)
-
     def sanitize_uri(
         self, uri_or_path_like: str
     ) -> tuple[AnyUrl, domain_entities.SourceType]:
@@ -297,7 +256,7 @@ class IndexDomainService:
                     await metadata_extractor.extract(file_path=file_path)
                 )
             except (OSError, ValueError) as e:
-                self.log.info("Skipping file", file=str(file_path), error=str(e))
+                self.log.debug("Skipping file", file=str(file_path), error=str(e))
                 continue
 
         # Finally check if there are any modified files

kodit/domain/value_objects.py

@@ -134,14 +134,6 @@ class SearchType(Enum):
     HYBRID = "hybrid"
 
 
-@dataclass
-class SnippetExtractionResult:
-    """Domain model for snippet extraction result."""
-
-    snippets: list[str]
-    language: str
-
-
 @dataclass
 class Document:
     """Generic document model for indexing."""
@@ -640,20 +632,6 @@ class SnippetQuery(BaseModel):
     top_k: int = 10
 
 
-class SnippetExtractionStrategy(str, Enum):
-    """Different strategies for extracting snippets from files."""
-
-    METHOD_BASED = "method_based"
-
-
-@dataclass
-class SnippetExtractionRequest:
-    """Domain model for snippet extraction request."""
-
-    file_path: Path
-    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED
-
-
 class FileProcessingStatus(IntEnum):
     """File processing status."""
 
@@ -661,3 +639,13 @@ class FileProcessingStatus(IntEnum):
     ADDED = 1
     MODIFIED = 2
     DELETED = 3
+
+
+@dataclass
+class FunctionDefinition:
+    """Cached function definition."""
+
+    name: str
+    qualified_name: str
+    start_byte: int
+    end_byte: int

kodit/infrastructure/slicing/__init__.py

@@ -0,0 +1 @@
+"""Slicing infrastructure module."""

kodit/infrastructure/slicing/language_detection_service.py

@@ -0,0 +1,18 @@
+"""Language detection service implementation."""
+
+from pathlib import Path
+
+from kodit.domain.services.index_service import LanguageDetectionService
+
+
+class FileSystemLanguageDetectionService(LanguageDetectionService):
+    """Simple file extension based language detection service."""
+
+    def __init__(self, language_map: dict[str, str]) -> None:
+        """Initialize with a mapping of extensions to languages."""
+        self._language_map = language_map
+
+    async def detect_language(self, file_path: Path) -> str:
+        """Detect language based on file extension."""
+        extension = file_path.suffix.lstrip(".")
+        return self._language_map.get(extension, "unknown")