kodit 0.3.8__tar.gz → 0.3.10__tar.gz

{kodit-0.3.8 → kodit-0.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.3.8
+Version: 0.3.10
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/

{kodit-0.3.8 → kodit-0.3.10}/src/kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.8'
-__version_tuple__ = version_tuple = (0, 3, 8)
+__version__ = version = '0.3.10'
+__version_tuple__ = version_tuple = (0, 3, 10)

{kodit-0.3.8 → kodit-0.3.10}/src/kodit/domain/services/index_service.py

@@ -1,6 +1,7 @@
 """Pure domain service for Index aggregate operations."""
 
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from pathlib import Path
 
 import structlog
@@ -104,30 +105,39 @@ class IndexDomainService:
 
         # Create a set of languages to extract snippets for
         extensions = {file.extension() for file in files}
-        languages = []
+        lang_files_map: dict[str, list[domain_entities.File]] = defaultdict(list)
         for ext in extensions:
             try:
-                languages.append(LanguageMapping.get_language_for_extension(ext))
+                lang = LanguageMapping.get_language_for_extension(ext)
+                lang_files_map[lang].extend(
+                    file for file in files if file.extension() == ext
+                )
             except ValueError as e:
-                self.log.info("Skipping", error=str(e))
+                self.log.debug("Skipping", error=str(e))
                 continue
 
+        self.log.info(
+            "Languages to process",
+            languages=lang_files_map.keys(),
+        )
+
         reporter = Reporter(self.log, progress_callback)
         await reporter.start(
             "extract_snippets",
-            len(files) * len(languages),
+            len(lang_files_map.keys()),
             "Extracting code snippets...",
         )
+
         # Calculate snippets for each language
         slicer = Slicer()
-        for i, language in enumerate(languages):
+        for i, (lang, lang_files) in enumerate(lang_files_map.items()):
             await reporter.step(
                 "extract_snippets",
-                len(files) * (i + 1),
-                len(files) * len(languages),
-                "Extracting code snippets...",
+                i,
+                len(lang_files_map.keys()),
+                f"Extracting code snippets for {lang}...",
             )
-            s = slicer.extract_snippets(files, language=language)
+            s = slicer.extract_snippets(lang_files, language=lang)
             index.snippets.extend(s)
 
         await reporter.done("extract_snippets")

{kodit-0.3.8 → kodit-0.3.10}/src/kodit/infrastructure/slicing/slicer.py

@@ -10,10 +10,12 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, ClassVar
 
+import structlog
 from tree_sitter import Node, Parser, Tree
 from tree_sitter_language_pack import get_language
 
 from kodit.domain.entities import File, Snippet
+from kodit.domain.value_objects import LanguageMapping
 
 
 @dataclass
@@ -145,8 +147,9 @@ class Slicer:
 
     def __init__(self) -> None:
         """Initialize an empty slicer."""
+        self.log = structlog.get_logger(__name__)
 
-    def extract_snippets(
+    def extract_snippets(  # noqa: C901
         self, files: list[File], language: str = "python"
     ) -> list[Snippet]:
         """Extract code snippets from a list of files.
@@ -170,6 +173,7 @@ class Slicer:
 
         # Get language configuration
         if language not in LanguageConfig.CONFIGS:
+            self.log.debug("Skipping", language=language)
             return []
 
         config = LanguageConfig.CONFIGS[language]
@@ -185,16 +189,20 @@ class Slicer:
         # Create mapping from Paths to File objects and extract paths
         path_to_file_map: dict[Path, File] = {}
         file_paths: list[Path] = []
-
         for file in files:
             file_path = file.as_path()
-            path_to_file_map[file_path] = file
-            file_paths.append(file_path)
+
+            # Validate file matches language
+            if not self._file_matches_language(file_path.suffix, language):
+                raise ValueError(f"File {file_path} does not match language {language}")
 
             # Validate file exists
             if not file_path.exists():
                 raise FileNotFoundError(f"File not found: {file_path}")
 
+            path_to_file_map[file_path] = file
+            file_paths.append(file_path)
+
         # Initialize state
         state = AnalyzerState(parser=parser)
         state.files = file_paths
@@ -209,7 +217,7 @@ class Slicer:
                 state.asts[file_path] = tree
             except OSError:
                 # Skip files that can't be parsed
-                pass
+                continue
 
         # Build indexes
         self._build_definition_and_import_indexes(state, config, language)
@@ -233,6 +241,19 @@ class Slicer:
 
         return snippets
 
+    def _file_matches_language(self, file_extension: str, language: str) -> bool:
+        """Check if a file extension matches the current language."""
+        if language not in LanguageConfig.CONFIGS:
+            return False
+
+        try:
+            return (
+                language == LanguageMapping.get_language_for_extension(file_extension)
+            )
+        except ValueError:
+            # Extension not supported, so it doesn't match any language
+            return False
+
     def _get_tree_sitter_language_name(self, language: str) -> str:
         """Map user language names to tree-sitter language names."""
         mapping = {
@@ -247,9 +268,9 @@ class Slicer:
             "typescript": "typescript",
             "js": "javascript",
             "ts": "typescript",
-            "csharp": "c_sharp",
-            "c#": "c_sharp",
-            "cs": "c_sharp",
+            "csharp": "csharp",
+            "c#": "csharp",
+            "cs": "csharp",
             "html": "html",
             "css": "css",
         }
@@ -299,19 +320,23 @@ class Slicer:
 
     def _walk_tree(self, node: Node) -> Generator[Node, None, None]:
         """Walk the AST tree, yielding all nodes."""
-        cursor = node.walk()
-
-        def _walk_recursive() -> Generator[Node, None, None]:
-            current_node = cursor.node
-            if current_node is not None:
-                yield current_node
-            if cursor.goto_first_child():
-                yield from _walk_recursive()
-                while cursor.goto_next_sibling():
-                    yield from _walk_recursive()
-                cursor.goto_parent()
-
-        yield from _walk_recursive()
+        # Use a simple queue-based approach to avoid recursion issues
+        queue = [node]
+        visited: set[int] = set()  # Track by node id (memory address)
+
+        while queue:
+            current = queue.pop(0)
+
+            # Use node id (memory address) as unique identifier to avoid infinite loops
+            node_id = id(current)
+            if node_id in visited:
+                continue
+            visited.add(node_id)
+
+            yield current
+
+            # Add children to queue
+            queue.extend(current.children)
 
     def _is_function_definition(self, node: Node, config: dict[str, Any]) -> bool:
         """Check if node is a function definition."""

kodit-0.3.10/tests/kodit/domain/services/index_service_test.py

@@ -0,0 +1,320 @@
+"""Unit tests for IndexDomainService."""
+
+from pathlib import Path
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from pydantic import AnyUrl
+
+from kodit.domain.entities import Index, Snippet, Source, SourceType, WorkingCopy
+from kodit.domain.services.enrichment_service import EnrichmentDomainService
+from kodit.domain.services.index_service import (
+    IndexDomainService,
+    LanguageDetectionService,
+)
+
+
+class MockLanguageDetectionService(LanguageDetectionService):
+    """Mock language detection service."""
+
+    async def detect_language(self, file_path: Path) -> str:
+        """Return a mock language based on file extension."""
+        if file_path.suffix == ".py":
+            return "python"
+        return "unknown"
+
+
+@pytest.fixture
+def mock_enrichment_service() -> EnrichmentDomainService:
+    """Create a mock enrichment service."""
+    service = Mock(spec=EnrichmentDomainService)
+    service.enrich_documents = AsyncMock()
+    return service
+
+
+@pytest.fixture
+def index_domain_service(
+    mock_enrichment_service: EnrichmentDomainService,
+    tmp_path: Path,
+) -> IndexDomainService:
+    """Create an IndexDomainService for testing."""
+    return IndexDomainService(
+        language_detector=MockLanguageDetectionService(),
+        enrichment_service=mock_enrichment_service,
+        clone_dir=tmp_path / "clones",
+    )
+
+
+@pytest.mark.asyncio
+async def test_prepare_index_creates_working_copy(
+    index_domain_service: IndexDomainService, tmp_path: Path
+) -> None:
+    """Test that prepare_index creates a working copy without repository calls."""
+    # Create a test file
+    test_file = tmp_path / "test.py"
+    test_file.write_text("def hello(): pass")
+
+    # Prepare index (this only creates the working copy structure, no files scanned yet)
+    working_copy = await index_domain_service.prepare_index(str(tmp_path))
+
+    # Verify the working copy was created
+    assert isinstance(working_copy, WorkingCopy)
+    assert isinstance(working_copy.remote_uri, AnyUrl)
+    assert working_copy.source_type == SourceType.FOLDER
+
+    # In the new flow, files are not scanned during prepare_index
+    # They are scanned during refresh_working_copy
+    assert len(working_copy.files) == 0
+
+    # Now refresh to actually scan the files
+    refreshed_working_copy = await index_domain_service.refresh_working_copy(
+        working_copy
+    )
+    assert len(refreshed_working_copy.files) == 1
+    assert refreshed_working_copy.files[0].uri.path.endswith("test.py")  # type: ignore[union-attr]
+
+
+@pytest.mark.asyncio
+async def test_extract_snippets_from_index_returns_snippets(
+    index_domain_service: IndexDomainService, tmp_path: Path
+) -> None:
+    """Test that extract_snippets_from_index returns snippets without persistence."""
+    # Create a mock index with files
+    test_file = tmp_path / "test.py"
+    test_file.write_text("def hello(): pass")
+
+    # Prepare working copy first
+    working_copy = await index_domain_service.prepare_index(str(tmp_path))
+
+    # Now refresh to scan the files
+    working_copy = await index_domain_service.refresh_working_copy(working_copy)
+
+    from datetime import UTC, datetime
+
+    # Create a mock index
+    source = Source(
+        id=1,
+        working_copy=working_copy,
+    )
+    index = Index(
+        id=1,
+        source=source,
+        snippets=[],
+        created_at=datetime.now(UTC),
+        updated_at=datetime.now(UTC),
+    )
+
+    # Extract snippets - this method returns the updated Index, not just snippets
+    updated_index = await index_domain_service.extract_snippets_from_index(
+        index=index,
+    )
+
+    # Verify snippets were extracted
+    assert len(updated_index.snippets) > 0
+    assert isinstance(updated_index.snippets[0], Snippet)
+    # The actual Slicer is used now, so we should check for the actual function
+    assert "def hello" in updated_index.snippets[0].original_text()
+    assert "pass" in updated_index.snippets[0].original_text()
+
+
+@pytest.mark.asyncio
+async def test_enrich_snippets_in_index_returns_enriched_snippets(
+    tmp_path: Path,
+) -> None:
+    """Test enrich_snippets_in_index returns enriched snippets without persistence."""
+    from kodit.domain.services.enrichment_service import EnrichmentDomainService
+    from kodit.infrastructure.enrichment.null_enrichment_provider import (
+        NullEnrichmentProvider,
+    )
+
+    # Create real enrichment service with null provider (fast)
+    enrichment_service = EnrichmentDomainService(
+        enrichment_provider=NullEnrichmentProvider()
+    )
+
+    # Create domain service with real enrichment service
+    domain_service = IndexDomainService(
+        language_detector=MockLanguageDetectionService(),
+        enrichment_service=enrichment_service,
+        clone_dir=tmp_path / "clones",
+    )
+
+    # Create mock snippets
+    snippet = Snippet(derives_from=[])
+    snippet.id = 1
+    snippet.add_original_content("def test(): pass", "python")
+    snippets = [snippet]
+
+    # Enrich snippets
+    enriched_snippets = await domain_service.enrich_snippets_in_index(snippets=snippets)
+
+    # Verify snippets were returned (null provider doesn't actually enrich)
+    assert len(enriched_snippets) == 1
+    assert enriched_snippets[0].id == 1
+
+
+@pytest.mark.asyncio
+async def test_enrich_snippets_with_empty_list_returns_empty_list(
+    index_domain_service: IndexDomainService,
+) -> None:
+    """Test that enriching an empty list returns an empty list."""
+    enriched_snippets = await index_domain_service.enrich_snippets_in_index(snippets=[])
+    assert enriched_snippets == []
+
+
+@pytest.mark.asyncio
+async def test_extract_snippets_only_processes_relevant_files(
+    index_domain_service: IndexDomainService, tmp_path: Path
+) -> None:
+    """Test that extract_snippets only processes files relevant to their language."""
+    # Create files of different types
+    py_file = tmp_path / "test.py"
+    py_file.write_text("def hello(): pass")
+
+    js_file = tmp_path / "test.js"
+    js_file.write_text("function hello() {}")
+
+    png_file = tmp_path / "image.png"
+    png_file.write_bytes(b"fake png data")
+
+    txt_file = tmp_path / "readme.txt"
+    txt_file.write_text("This is just text")
+
+    # Prepare working copy
+    working_copy = await index_domain_service.prepare_index(str(tmp_path))
+    working_copy = await index_domain_service.refresh_working_copy(working_copy)
+
+    # Verify all files were discovered
+    assert len(working_copy.files) == 4
+
+    from datetime import UTC, datetime
+
+    from kodit.domain.entities import Index, Source
+
+    # Create a mock index
+    source = Source(id=1, working_copy=working_copy)
+    index = Index(
+        id=1,
+        source=source,
+        snippets=[],
+        created_at=datetime.now(UTC),
+        updated_at=datetime.now(UTC),
+    )
+
+    # Extract snippets - should process all supported languages found in files
+    updated_index = await index_domain_service.extract_snippets_from_index(index)
+
+    # Should have snippets from both Python and JavaScript files (2 supported languages)
+    # PNG and TXT files should be ignored as they're not supported
+    assert len(updated_index.snippets) == 2  # Python and JavaScript files
+
+    # Get snippet texts
+    snippet_texts = [s.original_text() for s in updated_index.snippets]
+
+    # Should contain content from both supported languages
+    assert any("def hello" in text for text in snippet_texts)  # Python file
+    assert any("function hello" in text for text in snippet_texts)  # JavaScript file
+
+    # Verify the snippets derive from the correct files
+    source_files = [s.derives_from[0].uri.path for s in updated_index.snippets]
+    assert any(path.endswith("test.py") for path in source_files)  # type: ignore[union-attr]
+    assert any(path.endswith("test.js") for path in source_files)  # type: ignore[union-attr]
+
+
+@pytest.mark.asyncio
+async def test_extract_snippets_filters_by_language_mapping(
+    index_domain_service: IndexDomainService, tmp_path: Path
+) -> None:
+    """Test that extract_snippets correctly filters files by language mapping."""
+    # Create files for different languages
+    py_file = tmp_path / "script.py"
+    py_file.write_text("def python_func(): return 'python'")
+
+    js_file = tmp_path / "script.js"
+    js_file.write_text("function jsFunc() { return 'javascript'; }")
+
+    java_file = tmp_path / "Script.java"
+    java_file.write_text("public class Script { public void javaMethod() {} }")
+
+    # Create unsupported file type
+    unknown_file = tmp_path / "script.unknown"
+    unknown_file.write_text("some unknown content")
+
+    # Prepare working copy
+    working_copy = await index_domain_service.prepare_index(str(tmp_path))
+    working_copy = await index_domain_service.refresh_working_copy(working_copy)
+
+    # Should discover all files
+    assert len(working_copy.files) == 4
+
+    from datetime import UTC, datetime
+
+    from kodit.domain.entities import Index, Source
+
+    # Create a mock index
+    source = Source(id=1, working_copy=working_copy)
+    index = Index(
+        id=1,
+        source=source,
+        snippets=[],
+        created_at=datetime.now(UTC),
+        updated_at=datetime.now(UTC),
+    )
+
+    # Extract snippets - should process supported languages only
+    updated_index = await index_domain_service.extract_snippets_from_index(index)
+
+    # Should have snippets from Python, JavaScript, and Java files
+    # (unknown file should be ignored)
+    assert len(updated_index.snippets) == 3
+
+    # Get the snippet texts
+    snippet_texts = [s.original_text() for s in updated_index.snippets]
+
+    # Should contain content from all supported languages
+    assert any("def python_func" in text for text in snippet_texts)
+    assert any("function jsFunc" in text for text in snippet_texts)
+    assert any("javaMethod" in text for text in snippet_texts)
+
+    # Should not contain content from unsupported file
+    assert not any("unknown content" in text for text in snippet_texts)
+
+
+@pytest.mark.asyncio
+async def test_extract_snippets_handles_no_matching_files(
+    index_domain_service: IndexDomainService, tmp_path: Path
+) -> None:
+    """Test extract_snippets handles case where no files match supported languages."""
+    # Create only unsupported file types
+    img_file = tmp_path / "image.png"
+    img_file.write_bytes(b"fake image data")
+
+    doc_file = tmp_path / "document.pdf"
+    doc_file.write_bytes(b"fake pdf data")
+
+    # Prepare working copy
+    working_copy = await index_domain_service.prepare_index(str(tmp_path))
+    working_copy = await index_domain_service.refresh_working_copy(working_copy)
+
+    # Should discover files
+    assert len(working_copy.files) == 2
+
+    from datetime import UTC, datetime
+
+    from kodit.domain.entities import Index, Source
+
+    # Create a mock index
+    source = Source(id=1, working_copy=working_copy)
+    index = Index(
+        id=1,
+        source=source,
+        snippets=[],
+        created_at=datetime.now(UTC),
+        updated_at=datetime.now(UTC),
+    )
+
+    # Extract snippets - should return empty list since no supported files
+    updated_index = await index_domain_service.extract_snippets_from_index(index)
+
+    # Should have no snippets
+    assert len(updated_index.snippets) == 0