kodit 0.3.9__py3-none-any.whl → 0.3.10__py3-none-any.whl

kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.9'
-__version_tuple__ = version_tuple = (0, 3, 9)
+__version__ = version = '0.3.10'
+__version_tuple__ = version_tuple = (0, 3, 10)

kodit/domain/services/index_service.py

@@ -1,6 +1,7 @@
 """Pure domain service for Index aggregate operations."""
 
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from pathlib import Path
 
 import structlog
@@ -104,30 +105,39 @@ class IndexDomainService:
 
         # Create a set of languages to extract snippets for
         extensions = {file.extension() for file in files}
-        languages = []
+        lang_files_map: dict[str, list[domain_entities.File]] = defaultdict(list)
         for ext in extensions:
             try:
-                languages.append(LanguageMapping.get_language_for_extension(ext))
+                lang = LanguageMapping.get_language_for_extension(ext)
+                lang_files_map[lang].extend(
+                    file for file in files if file.extension() == ext
+                )
             except ValueError as e:
-                self.log.info("Skipping", error=str(e))
+                self.log.debug("Skipping", error=str(e))
                 continue
 
+        self.log.info(
+            "Languages to process",
+            languages=lang_files_map.keys(),
+        )
+
         reporter = Reporter(self.log, progress_callback)
         await reporter.start(
             "extract_snippets",
-            len(files) * len(languages),
+            len(lang_files_map.keys()),
             "Extracting code snippets...",
         )
+
         # Calculate snippets for each language
         slicer = Slicer()
-        for i, language in enumerate(languages):
+        for i, (lang, lang_files) in enumerate(lang_files_map.items()):
             await reporter.step(
                 "extract_snippets",
-                len(files) * (i + 1),
-                len(files) * len(languages),
-                "Extracting code snippets...",
+                i,
+                len(lang_files_map.keys()),
+                f"Extracting code snippets for {lang}...",
             )
-            s = slicer.extract_snippets(files, language=language)
+            s = slicer.extract_snippets(lang_files, language=lang)
             index.snippets.extend(s)
 
         await reporter.done("extract_snippets")

kodit/infrastructure/slicing/slicer.py

@@ -10,10 +10,12 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, ClassVar
 
+import structlog
 from tree_sitter import Node, Parser, Tree
 from tree_sitter_language_pack import get_language
 
 from kodit.domain.entities import File, Snippet
+from kodit.domain.value_objects import LanguageMapping
 
 
 @dataclass
@@ -145,8 +147,9 @@ class Slicer:
 
     def __init__(self) -> None:
         """Initialize an empty slicer."""
+        self.log = structlog.get_logger(__name__)
 
-    def extract_snippets(
+    def extract_snippets(  # noqa: C901
         self, files: list[File], language: str = "python"
     ) -> list[Snippet]:
         """Extract code snippets from a list of files.
@@ -170,6 +173,7 @@ class Slicer:
 
         # Get language configuration
         if language not in LanguageConfig.CONFIGS:
+            self.log.debug("Skipping", language=language)
             return []
 
         config = LanguageConfig.CONFIGS[language]
@@ -185,16 +189,20 @@ class Slicer:
         # Create mapping from Paths to File objects and extract paths
         path_to_file_map: dict[Path, File] = {}
         file_paths: list[Path] = []
-
         for file in files:
             file_path = file.as_path()
-            path_to_file_map[file_path] = file
-            file_paths.append(file_path)
+
+            # Validate file matches language
+            if not self._file_matches_language(file_path.suffix, language):
+                raise ValueError(f"File {file_path} does not match language {language}")
 
             # Validate file exists
             if not file_path.exists():
                 raise FileNotFoundError(f"File not found: {file_path}")
 
+            path_to_file_map[file_path] = file
+            file_paths.append(file_path)
+
         # Initialize state
         state = AnalyzerState(parser=parser)
         state.files = file_paths
@@ -209,7 +217,7 @@ class Slicer:
                 state.asts[file_path] = tree
             except OSError:
                 # Skip files that can't be parsed
-                pass
+                continue
 
         # Build indexes
         self._build_definition_and_import_indexes(state, config, language)
@@ -233,6 +241,19 @@ class Slicer:
 
         return snippets
 
+    def _file_matches_language(self, file_extension: str, language: str) -> bool:
+        """Check if a file extension matches the current language."""
+        if language not in LanguageConfig.CONFIGS:
+            return False
+
+        try:
+            return (
+                language == LanguageMapping.get_language_for_extension(file_extension)
+            )
+        except ValueError:
+            # Extension not supported, so it doesn't match any language
+            return False
+
     def _get_tree_sitter_language_name(self, language: str) -> str:
         """Map user language names to tree-sitter language names."""
         mapping = {
@@ -299,19 +320,23 @@ class Slicer:
 
     def _walk_tree(self, node: Node) -> Generator[Node, None, None]:
         """Walk the AST tree, yielding all nodes."""
-        cursor = node.walk()
-
-        def _walk_recursive() -> Generator[Node, None, None]:
-            current_node = cursor.node
-            if current_node is not None:
-                yield current_node
-            if cursor.goto_first_child():
-                yield from _walk_recursive()
-                while cursor.goto_next_sibling():
-                    yield from _walk_recursive()
-                cursor.goto_parent()
-
-        yield from _walk_recursive()
+        # Use a simple queue-based approach to avoid recursion issues
+        queue = [node]
+        visited: set[int] = set()  # Track by node id (memory address)
+
+        while queue:
+            current = queue.pop(0)
+
+            # Use node id (memory address) as unique identifier to avoid infinite loops
+            node_id = id(current)
+            if node_id in visited:
+                continue
+            visited.add(node_id)
+
+            yield current
+
+            # Add children to queue
+            queue.extend(current.children)
 
     def _is_function_definition(self, node: Node, config: dict[str, Any]) -> bool:
         """Check if node is a function definition."""

{kodit-0.3.9.dist-info → kodit-0.3.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.3.9
+Version: 0.3.10
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/

{kodit-0.3.9.dist-info → kodit-0.3.10.dist-info}/RECORD

@@ -1,6 +1,6 @@
 kodit/.gitignore,sha256=ztkjgRwL9Uud1OEi36hGQeDGk3OLK1NfDEO8YqGYy8o,11
 kodit/__init__.py,sha256=aEKHYninUq1yh6jaNfvJBYg-6fenpN132nJt1UU6Jxs,59
-kodit/_version.py,sha256=2CFg1ZfWyy1OjlshL7TqqlWTw1jORYgWRqP2Fc0c4K0,511
+kodit/_version.py,sha256=8f9qESpn_-snEACtTM18TNc6AEJLWU6rpXlL21ijVSc,513
 kodit/app.py,sha256=3_smkoioIQEYtRLIGHDtgGkmkP6Movd5CygQEMOStP8,3043
 kodit/cli.py,sha256=ZOS_VzCHGjJRZzZpaVR00QXSPIwRXPYu-pTrbEtlyR0,19328
 kodit/config.py,sha256=Il_eeyg7s83QF5lmiFB6qX6pmpiqCWncHtPgPcdA4xA,8063
@@ -26,7 +26,7 @@ kodit/domain/services/bm25_service.py,sha256=nsfTan3XtDwXuuAu1LUv-6Jukm6qFKVqqCV
 kodit/domain/services/embedding_service.py,sha256=7drYRC2kjg0WJmo06a2E9N0vDnwInUlBB96twjz2BT8,4526
 kodit/domain/services/enrichment_service.py,sha256=XsXg3nV-KN4rqtC7Zro_ZiZ6RSq-1eA1MG6IDzFGyBA,1316
 kodit/domain/services/index_query_service.py,sha256=02UWfyB_HoHUskunGuHeq5XwQLSWxGSK4OhvxcqIfY0,2022
-kodit/domain/services/index_service.py,sha256=ezVGbWdII25adri4_yyvsAF2eJOt4xmoHRDuS_-c6Ro,10810
+kodit/domain/services/index_service.py,sha256=r6skJzN0Hp_lJNaUjQSpHSRETCHNnfmJWH4X6A2-rFE,11159
 kodit/infrastructure/__init__.py,sha256=HzEYIjoXnkz_i_MHO2e0sIVYweUcRnl2RpyBiTbMObU,28
 kodit/infrastructure/bm25/__init__.py,sha256=DmGbrEO34FOJy4e685BbyxLA7gPW1eqs2gAxsp6JOuM,34
 kodit/infrastructure/bm25/bm25_factory.py,sha256=I4eo7qRslnyXIRkBf-StZ5ga2Evrr5J5YFocTChFD3g,884
@@ -61,7 +61,7 @@ kodit/infrastructure/mappers/__init__.py,sha256=QPHOjNreXmBPPovZ6elnYFS0vD-IsmrG
 kodit/infrastructure/mappers/index_mapper.py,sha256=ZSfu8kjTaa8_UY0nTqr4b02NS3VrjqZYkduCN71AL2g,12743
 kodit/infrastructure/slicing/__init__.py,sha256=x7cjvHA9Ay2weUYE_dpdAaPaStp20M-4U2b5MLgT5KM,37
 kodit/infrastructure/slicing/language_detection_service.py,sha256=JGJXrq9bLyfnisWJXeP7y1jbZMmKAISdPBlRBCosUcE,684
-kodit/infrastructure/slicing/slicer.py,sha256=HGpxbMJWoDXhLLC7RJzWLAuE93zpoVW1MXzntJhueVo,34626
+kodit/infrastructure/slicing/slicer.py,sha256=GOqJykd00waOTO1WJHyE5KUgJ2RLx2rOQ7M7T_u5LLg,35600
 kodit/infrastructure/sqlalchemy/__init__.py,sha256=UXPMSF_hgWaqr86cawRVqM8XdVNumQyyK5B8B97GnlA,33
 kodit/infrastructure/sqlalchemy/embedding_repository.py,sha256=dC2Wzj_zQiWExwfScE1LAGiiyxPyg0YepwyLOgDwcs4,7905
 kodit/infrastructure/sqlalchemy/entities.py,sha256=Dmh0z-dMI0wfMAPpf62kxU4md6NUH9P5Nx1QSTITOfg,5961
@@ -82,8 +82,8 @@ kodit/migrations/versions/__init__.py,sha256=9-lHzptItTzq_fomdIRBegQNm4Znx6pVjwD
 kodit/migrations/versions/c3f5137d30f5_index_all_the_things.py,sha256=r7ukmJ_axXLAWewYx-F1fEmZ4JbtFd37i7cSb0tq3y0,1722
 kodit/utils/__init__.py,sha256=DPEB1i8evnLF4Ns3huuAYg-0pKBFKUFuiDzOKG9r-sw,33
 kodit/utils/path_utils.py,sha256=thK6YGGNvQThdBaCYCCeCvS1L8x-lwl3AoGht2jnjGw,1645
-kodit-0.3.9.dist-info/METADATA,sha256=3YOAFUcvQJJSNSOfZvhfp-VcAmQbwUjbZRwmz1qXekI,6973
-kodit-0.3.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kodit-0.3.9.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
-kodit-0.3.9.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-kodit-0.3.9.dist-info/RECORD,,
+kodit-0.3.10.dist-info/METADATA,sha256=SUpOyQI6dJQnd9Mza8Dml3A4zOeoM_XV6Q6cac2k3rw,6974
+kodit-0.3.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kodit-0.3.10.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
+kodit-0.3.10.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+kodit-0.3.10.dist-info/RECORD,,