kodit 0.5.6__py3-none-any.whl → 0.5.7__py3-none-any.whl

kodit/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.5.6'
-__version_tuple__ = version_tuple = (0, 5, 6)
+__version__ = version = '0.5.7'
+__version_tuple__ = version_tuple = (0, 5, 7)
 
 __commit_id__ = commit_id = None

kodit/application/services/commit_indexing_application_service.py

@@ -37,7 +37,13 @@ from kodit.domain.enrichments.request import (
     EnrichmentRequest as GenericEnrichmentRequest,
 )
 from kodit.domain.entities import Task
-from kodit.domain.entities.git import GitFile, GitRepo, SnippetV2, TrackingType
+from kodit.domain.entities.git import (
+    GitCommit,
+    GitFile,
+    GitRepo,
+    SnippetV2,
+    TrackingType,
+)
 from kodit.domain.factories.git_repo_factory import GitRepoFactory
 from kodit.domain.protocols import (
     EnrichmentAssociationRepository,
@@ -278,6 +284,57 @@ class CommitIndexingApplicationService:
         else:
             raise ValueError(f"Unknown task type: {task.type}")
 
+    async def _process_files_in_batches(
+        self, cloned_path: Path, all_commits: list[GitCommit], batch_size: int = 100
+    ) -> int:
+        """Process file metadata for all commits in batches to avoid memory exhaustion.
+
+        This loads file metadata (paths, sizes, blob SHAs) in batches and saves them
+        incrementally to avoid holding millions of file objects in memory.
+
+        Args:
+            cloned_path: Path to the cloned repository
+            all_commits: List of all commits from scan
+            batch_size: Number of commits to process at once (default 100)
+
+        Returns:
+            Total number of files processed
+
+        """
+        total_files = 0
+        commit_shas = [commit.commit_sha for commit in all_commits]
+        total_batches = (len(commit_shas) + batch_size - 1) // batch_size
+
+        self._log.info(
+            f"Processing files for {len(commit_shas)} commits "
+            f"in {total_batches} batches"
+        )
+
+        # Process commits in batches
+        for i in range(0, len(commit_shas), batch_size):
+            batch = commit_shas[i : i + batch_size]
+            batch_num = i // batch_size + 1
+
+            self._log.debug(
+                f"Processing batch {batch_num}/{total_batches} ({len(batch)} commits)"
+            )
+
+            # Get file metadata for this batch of commits
+            files = await self.scanner.process_files_for_commits_batch(
+                cloned_path, batch
+            )
+
+            # Save file metadata to database immediately
+            if files:
+                await self.git_file_repository.save_bulk(files)
+                total_files += len(files)
+                self._log.debug(
+                    f"Batch {batch_num}: Saved {len(files)} files "
+                    f"(total so far: {total_files})"
+                )
+
+        return total_files
+
     async def process_clone_repo(self, repository_id: int) -> None:
         """Clone a repository."""
         async with self.operation.create_child(
@@ -315,8 +372,11 @@ class CommitIndexingApplicationService:
             await step.set_current(2, "Saving commits")
             await self.git_commit_repository.save_bulk(scan_result.all_commits)
 
-            await step.set_current(3, "Saving files")
-            await self.git_file_repository.save_bulk(scan_result.all_files)
+            await step.set_current(3, "Processing and saving files in batches")
+            total_files = await self._process_files_in_batches(
+                repo.cloned_path, scan_result.all_commits
+            )
+            self._log.info(f"Processed and saved {total_files} total files")
 
             await step.set_current(4, "Saving branches")
             if scan_result.branches:

kodit/domain/protocols.py

@@ -4,6 +4,8 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Protocol, TypeVar
 
+from git import Repo
+
 from kodit.domain.enrichments.enrichment import EnrichmentAssociation, EnrichmentV2
 from kodit.domain.entities import (
     Task,
@@ -163,9 +165,16 @@ class GitAdapter(ABC):
 
     @abstractmethod
     async def get_commit_files(
-        self, local_path: Path, commit_sha: str
+        self, local_path: Path, commit_sha: str, repo: Repo
     ) -> list[dict[str, Any]]:
-        """Get all files in a specific commit from the git tree."""
+        """Get all files in a specific commit from the git tree.
+
+        Args:
+            local_path: Path to the repository
+            commit_sha: SHA of the commit to get files for
+            repo: Repo object to reuse (avoids creating new Repo per commit)
+
+        """
 
     @abstractmethod
     async def get_commit_file_data(

kodit/domain/services/git_repository_service.py

@@ -1,6 +1,5 @@
 """Domain services for Git repository scanning and cloning operations."""
 
-import asyncio
 import shutil
 from dataclasses import dataclass
 from datetime import UTC, datetime
@@ -66,51 +65,11 @@ class GitRepositoryScanner:
         tags = await self._process_tags(cloned_path, commit_cache, repo_id)
         self._log.info(f"Found {len(tags)} tags")
 
-        all_files = await self._process_files(cloned_path, commit_cache)
-        self._log.info(f"Found {len(all_files)} files")
+        # Don't load all files into memory - return empty list
+        # Files will be processed in batches by the application service
+        self._log.info("Deferring file processing to avoid memory exhaustion")
 
-        return self._create_scan_result(branches, commit_cache, tags, all_files)
-
-    async def _process_commits_concurrently(
-        self,
-        cloned_path: Path,
-        commits_batch: list[tuple[str, dict[str, Any]]],
-    ) -> dict[str, GitCommit]:
-        """Process a batch of commits concurrently."""
-        batch_cache = {}
-
-        async def process_single_commit(
-            commit_sha: str, commit_data: dict[str, Any]
-        ) -> tuple[str, GitCommit | None]:
-            git_commit = await self._create_git_commit_from_data(
-                cloned_path, commit_data
-            )
-            return commit_sha, git_commit
-
-        # Process commits concurrently in smaller batches
-        semaphore = asyncio.Semaphore(50)  # Limit concurrent operations
-
-        async def bounded_process(
-            item: tuple[str, dict[str, Any]],
-        ) -> tuple[str, GitCommit | None]:
-            async with semaphore:
-                return await process_single_commit(item[0], item[1])
-
-        # Process all commits concurrently
-        results = await asyncio.gather(
-            *[bounded_process(item) for item in commits_batch],
-            return_exceptions=True,
-        )
-
-        # Collect successful results
-        for result in results:
-            if isinstance(result, tuple):
-                # Type narrowing: result is now tuple[str, GitCommit | None]
-                commit_sha, git_commit = result
-                if git_commit is not None:
-                    batch_cache[commit_sha] = git_commit
-
-        return batch_cache
+        return self._create_scan_result(branches, commit_cache, tags, [], cloned_path)
 
     async def _process_branches_bulk(
         self,
@@ -167,30 +126,6 @@ class GitRepositoryScanner:
 
         return branches, commit_cache
 
-    async def _create_git_commit_from_data(
-        self, cloned_path: Path, commit_data: dict[str, Any], repo_id: int | None = None
-    ) -> GitCommit | None:
-        """Create GitCommit from pre-fetched commit data."""
-        commit_sha = commit_data["sha"]
-
-        # Get files for this commit
-        files_data = await self.git_adapter.get_commit_files(cloned_path, commit_sha)
-        self._create_git_files(cloned_path, files_data, commit_sha)
-        author = self._format_author_from_data(commit_data)
-
-        # Cache datetime creation
-        created_at = datetime.now(UTC)
-
-        return GitCommit(
-            created_at=created_at,
-            commit_sha=commit_sha,
-            repo_id=repo_id or 0,  # Use 0 as default if not provided
-            date=commit_data["date"],
-            message=commit_data["message"],
-            parent_commit_sha=commit_data["parent_sha"],
-            author=author,
-        )
-
     def _format_author_from_data(self, commit_data: dict[str, Any]) -> str:
         """Format author string from commit data."""
         author_name = commit_data.get("author_name", "")
@@ -376,17 +311,18 @@ class GitRepositoryScanner:
         branches: list[GitBranch],
         commit_cache: dict[str, GitCommit],
         tags: list[GitTag],
-        all_files: list[GitFile],
+        all_files: list[GitFile],  # noqa: ARG002
+        cloned_path: Path | None = None,  # noqa: ARG002
     ) -> RepositoryScanResult:
         """Create final scan result."""
-        # Files are loaded on-demand for performance, so total_files is 0 during scan
+        # Files list is empty to avoid memory issues - will be processed in batches
         scan_result = RepositoryScanResult(
             branches=branches,
             all_commits=list(commit_cache.values()),
             scan_timestamp=datetime.now(UTC),
-            total_files_across_commits=len(all_files),
+            total_files_across_commits=0,  # Will be updated after batch processing
             all_tags=tags,
-            all_files=all_files,
+            all_files=[],  # Empty - processed in batches to avoid memory exhaustion
         )
 
         self._log.info(
@@ -395,16 +331,35 @@ class GitRepositoryScanner:
         )
         return scan_result
 
-    async def _process_files(
-        self, cloned_path: Path, commit_cache: dict[str, GitCommit]
+    async def process_files_for_commits_batch(
+        self, cloned_path: Path, commit_shas: list[str]
     ) -> list[GitFile]:
-        """Process files for a commit."""
+        """Process files for a batch of commits.
+
+        This allows the application service to process files in batches
+        to avoid loading millions of files into memory at once.
+
+        CRITICAL: Reuses a single Repo object to avoid creating 32K+ Repo instances
+        which would consume massive memory (1-2 MB each).
+        """
+        from git import Repo
+
+        # Open repo once and reuse for all commits in this batch
+        repo = Repo(cloned_path)
         files = []
-        for commit_sha in commit_cache:
-            files_data = await self.git_adapter.get_commit_files(
-                cloned_path, commit_sha
-            )
-            files.extend(self._create_git_files(cloned_path, files_data, commit_sha))
+
+        try:
+            for commit_sha in commit_shas:
+                files_data = await self.git_adapter.get_commit_files(
+                    cloned_path, commit_sha, repo=repo
+                )
+                files.extend(
+                    self._create_git_files(cloned_path, files_data, commit_sha)
+                )
+        finally:
+            # Explicitly close the repo to free resources
+            repo.close()
+
         return files
 
 

kodit/infrastructure/cloning/git/git_python_adaptor.py

@@ -346,14 +346,22 @@ class GitPythonAdapter(GitAdapter):
         )
 
     async def get_commit_files(
-        self, local_path: Path, commit_sha: str
+        self, local_path: Path, commit_sha: str, repo: Repo
     ) -> list[dict[str, Any]]:
-        """Get all files in a specific commit from the git tree."""
+        """Get all files in a specific commit from the git tree.
+
+        Args:
+            local_path: Path to the repository
+            commit_sha: SHA of the commit to get files for
+            repo: Repo object to reuse (avoids creating new Repo per commit)
+
+        """
 
         def _get_files() -> list[dict[str, Any]]:
             try:
-                repo = Repo(local_path)
-                commit = repo.commit(commit_sha)
+                # Use the provided repo object
+                _repo = repo
+                commit = _repo.commit(commit_sha)
 
                 files = []
 
@@ -395,7 +403,11 @@ class GitPythonAdapter(GitAdapter):
         """Get file metadata for a commit, with files checked out to disk."""
         await self._checkout_commit(local_path, commit_sha)
         try:
-            return await self.get_commit_files(local_path, commit_sha)
+            repo = Repo(local_path)
+            try:
+                return await self.get_commit_files(local_path, commit_sha, repo)
+            finally:
+                repo.close()
         finally:
             await self.restore_to_branch(local_path, "main")
 
@@ -570,6 +582,4 @@ class GitPythonAdapter(GitAdapter):
             else:
                 return diff_text
 
-        return await asyncio.get_event_loop().run_in_executor(
-            self.executor, _get_diff
-        )
+        return await asyncio.get_event_loop().run_in_executor(self.executor, _get_diff)

{kodit-0.5.6.dist-info → kodit-0.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.5.6
+Version: 0.5.7
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/

{kodit-0.5.6.dist-info → kodit-0.5.7.dist-info}/RECORD

@@ -1,6 +1,6 @@
 kodit/.gitignore,sha256=ztkjgRwL9Uud1OEi36hGQeDGk3OLK1NfDEO8YqGYy8o,11
 kodit/__init__.py,sha256=aEKHYninUq1yh6jaNfvJBYg-6fenpN132nJt1UU6Jxs,59
-kodit/_version.py,sha256=cK44j3gESPDerYNsAlInZd36FaFcP1dYcSF1jeRDXF8,704
+kodit/_version.py,sha256=NvV7p6eu_Rli4DWHJnEcpyTUiImNPPDyoDonzzIsNwA,704
 kodit/app.py,sha256=7WxSQcktnpYBmjO1skIjMeBu55rVVRf4lotBEq55pAM,5846
 kodit/cli.py,sha256=QSTXIUDxZo3anIONY-grZi9_VSehWoS8QoVJZyOmWPQ,3086
 kodit/cli_utils.py,sha256=umkvt4kWNapk6db6RGz6bmn7oxgDpsW2Vo09MZ37OGg,2430
@@ -16,7 +16,7 @@ kodit/application/factories/reporting_factory.py,sha256=3IpRiAw_olM69db-jbDAtjyG
 kodit/application/factories/server_factory.py,sha256=dr0X_zQRUlEybtGZ3NS-kkwTU-K96u2D1Qw5xhWkd88,17409
 kodit/application/services/__init__.py,sha256=p5UQNw-H5sxQvs5Etfte93B3cJ1kKW6DNxK34uFvU1E,38
 kodit/application/services/code_search_application_service.py,sha256=ceyv5TTN-jvlOFOupGa9XwfTTraLNN2GU55kFeulVXY,7763
-kodit/application/services/commit_indexing_application_service.py,sha256=CIJdBdAIdere6fEJ1KytQgDu0jNXoDiE748aFcEiOmQ,42631
+kodit/application/services/commit_indexing_application_service.py,sha256=uRYPkVbiqu1V9bORjQu2yoylskLgCz55vYJ1pODjm94,44690
 kodit/application/services/enrichment_query_service.py,sha256=RMVze-DzS5zAki1iC96Kid7tbg-nHSv0z8eqPsiURqc,15002
 kodit/application/services/indexing_worker_service.py,sha256=59cZthlzViOVrAWEoZqUTCfLzxx2OO_FOGdM3pYf9Mc,4065
 kodit/application/services/queue_service.py,sha256=pIHTS8M65FzAhZH5kn54BTiZ43sCbsALYdCFTz9wdqE,2692
@@ -24,7 +24,7 @@ kodit/application/services/reporting.py,sha256=cwe-S-UpSOE6xSAEhoD1hi4hSWk1bW3YR
 kodit/application/services/sync_scheduler.py,sha256=hVT3dlmvfbqXKOV_KU5ZQ5gEKBGPJTlvJcF9gP2ZHQM,2853
 kodit/domain/__init__.py,sha256=TCpg4Xx-oF4mKV91lo4iXqMEfBT1OoRSYnbG-zVWolA,66
 kodit/domain/errors.py,sha256=yIsgCjM_yOFIg8l7l-t7jM8pgeAX4cfPq0owf7iz3DA,106
-kodit/domain/protocols.py,sha256=KxTHnYbECvMoXdbvXNHVF-l-OEr2kzQsoxlVXbatgwg,7814
+kodit/domain/protocols.py,sha256=Q6blYD79Tn5LQyNEAioTuPPIdZYXDf46kVpAW2EG2jY,8056
 kodit/domain/value_objects.py,sha256=FW0sTMtcl0Q1qej7vzEg7-Gsv86Z01IbPrDdudsgU3g,18097
 kodit/domain/enrichments/__init__.py,sha256=UpQMnMEHqaK3u3K-eJZOC28kfBPHALLAjFMdyYBXSPE,33
 kodit/domain/enrichments/enricher.py,sha256=jnZ5X9RmZA8Acy-RBS2TbEoBg9QSm8AgleqwS9h5WlY,512
@@ -57,7 +57,7 @@ kodit/domain/factories/git_repo_factory.py,sha256=EdeQo4HsBi2hVeVvnSnYtFdR3yGVZQ
 kodit/domain/services/__init__.py,sha256=Q1GhCK_PqKHYwYE4tkwDz5BIyXkJngLBBOHhzvX8nzo,42
 kodit/domain/services/bm25_service.py,sha256=-E5k0td2Ucs25qygWkJlY0fl7ZckOUe5xZnKYff3hF8,3631
 kodit/domain/services/embedding_service.py,sha256=CEcQ2E9XvOcjKNCJEw5soYUNMHJ5LCJGyXzPCl75CPc,4812
-kodit/domain/services/git_repository_service.py,sha256=suIBmiBG9OcXUFrw1uiYRidS9yvFEekZU8H-tsY0zs0,16545
+kodit/domain/services/git_repository_service.py,sha256=KtwYF3XKBeNbAHbi-sEdMJ-1jGRy7rmWMZkPpCrh9fw,14980
 kodit/domain/services/git_service.py,sha256=Lr7kPnnBEa_fWfGA9jpffMK7wcfxQ0wfXgynsbSKSzg,11661
 kodit/domain/services/physical_architecture_service.py,sha256=0YgoAvbUxT_VwgIh_prftSYnil_XIqNPSoP0g37eIt4,7209
 kodit/domain/services/task_status_query_service.py,sha256=rI93pTMHeycigQryCWkimXSDzRqx_nJOr07UzPAacPE,736
@@ -95,7 +95,7 @@ kodit/infrastructure/bm25/__init__.py,sha256=DmGbrEO34FOJy4e685BbyxLA7gPW1eqs2gA
 kodit/infrastructure/bm25/local_bm25_repository.py,sha256=YE3pUkPS5n1JNu6oSM_HRBOXM8U04HiY8dMMZCf9CMQ,5197
 kodit/infrastructure/bm25/vectorchord_bm25_repository.py,sha256=LjbUPj4nPMb9pdEudThUbZTmQjhxvpN314EzKGpXfi0,8621
 kodit/infrastructure/cloning/git/__init__.py,sha256=20ePcp0qE6BuLsjsv4KYB1DzKhMIMsPXwEqIEZtjTJs,34
-kodit/infrastructure/cloning/git/git_python_adaptor.py,sha256=GCBdswC5txbt8_TWbMEbhvRvlmOvPCLwb6qBlx7b3XU,21594
+kodit/infrastructure/cloning/git/git_python_adaptor.py,sha256=kiiXrjSqdSYT_c_migWff1WEVlJT8JRlgo5m_9T4rrM,21942
 kodit/infrastructure/cloning/git/working_copy.py,sha256=sPKQN-A1gDVV_QJISNNP4PqxRWxyj5owv5tvWfXMl44,3909
 kodit/infrastructure/database_schema/__init__.py,sha256=jgejYX70fjV69zCuOBiNw3oCQlCKYzxTkjnUUUU7DY0,48
 kodit/infrastructure/database_schema/database_schema_detector.py,sha256=zXU7HqrZU4_EYckloKDbH0gZvZ3_TJG5-Bd5PAkEkXc,10167
@@ -177,8 +177,8 @@ kodit/utils/dump_config.py,sha256=dd5uPgqh6ATk02Zt59t2JFKR9X17YWjHudV0nE8VktE,11
 kodit/utils/dump_openapi.py,sha256=EasYOnnpeabwb_sTKQUBrrOLHjPcOFQ7Zx0YKpx9fmM,1239
 kodit/utils/generate_api_paths.py,sha256=TMtx9v55podDfUmiWaHgJHLtEWLV2sLL-5ejGFMPzAo,3569
 kodit/utils/path_utils.py,sha256=UB_81rx7Y1G1jalVv2PX8miwaprBbcqEdtoQ3hPT3kU,2451
-kodit-0.5.6.dist-info/METADATA,sha256=7G7BfnWWbmDkwSSiKribmesiUY5NlzjcxwqsZiTJjUw,7703
-kodit-0.5.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kodit-0.5.6.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
-kodit-0.5.6.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-kodit-0.5.6.dist-info/RECORD,,
+kodit-0.5.7.dist-info/METADATA,sha256=Yi8IGWrrk1FLgnC5GiqmBc8V3bJcWz8Fl29-nM8CkcE,7703
+kodit-0.5.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kodit-0.5.7.dist-info/entry_points.txt,sha256=hoTn-1aKyTItjnY91fnO-rV5uaWQLQ-Vi7V5et2IbHY,40
+kodit-0.5.7.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+kodit-0.5.7.dist-info/RECORD,,