kodit 0.5.6__tar.gz → 0.5.7__tar.gz

{kodit-0.5.6 → kodit-0.5.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.5.6
+Version: 0.5.7
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/

{kodit-0.5.6 → kodit-0.5.7}/docs/reference/api/index.md

@@ -12,7 +12,7 @@ look at the [hosted version](https://kodit.helix.ml/docs).
 This is the REST API for the Kodit server. Please refer to the
 [Kodit documentation](https://docs.helix.ml/kodit/) for more information.
     
-Current version: 0.5.5
+Current version: 0.5.6
 
 ## Authentication
 

{kodit-0.5.6 → kodit-0.5.7}/docs/reference/api/openapi.json

@@ -3,7 +3,7 @@
   "info": {
     "title": "kodit API",
     "description": "\nThis is the REST API for the Kodit server. Please refer to the\n[Kodit documentation](https://docs.helix.ml/kodit/) for more information.\n    ",
-    "version": "0.5.5"
+    "version": "0.5.6"
   },
   "paths": {
     "/healthz": {
@@ -2496,7 +2496,9 @@
           "kodit.commit.create_summary_enrichment",
           "kodit.commit.create_summary_embeddings",
           "kodit.commit.create_architecture_enrichment",
-          "kodit.commit.create_public_api_docs"
+          "kodit.commit.create_public_api_docs",
+          "kodit.commit.create_commit_description",
+          "kodit.commit.create_database_schema"
         ],
         "title": "TaskOperation",
         "description": "Task operation."

{kodit-0.5.6 → kodit-0.5.7}/src/kodit/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.5.6'
-__version_tuple__ = version_tuple = (0, 5, 6)
+__version__ = version = '0.5.7'
+__version_tuple__ = version_tuple = (0, 5, 7)
 
 __commit_id__ = commit_id = None

{kodit-0.5.6 → kodit-0.5.7}/src/kodit/application/services/commit_indexing_application_service.py

@@ -37,7 +37,13 @@ from kodit.domain.enrichments.request import (
     EnrichmentRequest as GenericEnrichmentRequest,
 )
 from kodit.domain.entities import Task
-from kodit.domain.entities.git import GitFile, GitRepo, SnippetV2, TrackingType
+from kodit.domain.entities.git import (
+    GitCommit,
+    GitFile,
+    GitRepo,
+    SnippetV2,
+    TrackingType,
+)
 from kodit.domain.factories.git_repo_factory import GitRepoFactory
 from kodit.domain.protocols import (
     EnrichmentAssociationRepository,
@@ -278,6 +284,57 @@ class CommitIndexingApplicationService:
         else:
             raise ValueError(f"Unknown task type: {task.type}")
 
+    async def _process_files_in_batches(
+        self, cloned_path: Path, all_commits: list[GitCommit], batch_size: int = 100
+    ) -> int:
+        """Process file metadata for all commits in batches to avoid memory exhaustion.
+
+        This loads file metadata (paths, sizes, blob SHAs) in batches and saves them
+        incrementally to avoid holding millions of file objects in memory.
+
+        Args:
+            cloned_path: Path to the cloned repository
+            all_commits: List of all commits from scan
+            batch_size: Number of commits to process at once (default 100)
+
+        Returns:
+            Total number of files processed
+
+        """
+        total_files = 0
+        commit_shas = [commit.commit_sha for commit in all_commits]
+        total_batches = (len(commit_shas) + batch_size - 1) // batch_size
+
+        self._log.info(
+            f"Processing files for {len(commit_shas)} commits "
+            f"in {total_batches} batches"
+        )
+
+        # Process commits in batches
+        for i in range(0, len(commit_shas), batch_size):
+            batch = commit_shas[i : i + batch_size]
+            batch_num = i // batch_size + 1
+
+            self._log.debug(
+                f"Processing batch {batch_num}/{total_batches} ({len(batch)} commits)"
+            )
+
+            # Get file metadata for this batch of commits
+            files = await self.scanner.process_files_for_commits_batch(
+                cloned_path, batch
+            )
+
+            # Save file metadata to database immediately
+            if files:
+                await self.git_file_repository.save_bulk(files)
+                total_files += len(files)
+                self._log.debug(
+                    f"Batch {batch_num}: Saved {len(files)} files "
+                    f"(total so far: {total_files})"
+                )
+
+        return total_files
+
     async def process_clone_repo(self, repository_id: int) -> None:
         """Clone a repository."""
         async with self.operation.create_child(
@@ -315,8 +372,11 @@ class CommitIndexingApplicationService:
             await step.set_current(2, "Saving commits")
             await self.git_commit_repository.save_bulk(scan_result.all_commits)
 
-            await step.set_current(3, "Saving files")
-            await self.git_file_repository.save_bulk(scan_result.all_files)
+            await step.set_current(3, "Processing and saving files in batches")
+            total_files = await self._process_files_in_batches(
+                repo.cloned_path, scan_result.all_commits
+            )
+            self._log.info(f"Processed and saved {total_files} total files")
 
             await step.set_current(4, "Saving branches")
             if scan_result.branches:

{kodit-0.5.6 → kodit-0.5.7}/src/kodit/domain/protocols.py

@@ -4,6 +4,8 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Protocol, TypeVar
 
+from git import Repo
+
 from kodit.domain.enrichments.enrichment import EnrichmentAssociation, EnrichmentV2
 from kodit.domain.entities import (
     Task,
@@ -163,9 +165,16 @@ class GitAdapter(ABC):
 
     @abstractmethod
     async def get_commit_files(
-        self, local_path: Path, commit_sha: str
+        self, local_path: Path, commit_sha: str, repo: Repo
     ) -> list[dict[str, Any]]:
-        """Get all files in a specific commit from the git tree."""
+        """Get all files in a specific commit from the git tree.
+
+        Args:
+            local_path: Path to the repository
+            commit_sha: SHA of the commit to get files for
+            repo: Repo object to reuse (avoids creating new Repo per commit)
+
+        """
 
     @abstractmethod
     async def get_commit_file_data(

{kodit-0.5.6 → kodit-0.5.7}/src/kodit/domain/services/git_repository_service.py

@@ -1,6 +1,5 @@
 """Domain services for Git repository scanning and cloning operations."""
 
-import asyncio
 import shutil
 from dataclasses import dataclass
 from datetime import UTC, datetime
@@ -66,51 +65,11 @@ class GitRepositoryScanner:
         tags = await self._process_tags(cloned_path, commit_cache, repo_id)
         self._log.info(f"Found {len(tags)} tags")
 
-        all_files = await self._process_files(cloned_path, commit_cache)
-        self._log.info(f"Found {len(all_files)} files")
+        # Don't load all files into memory - return empty list
+        # Files will be processed in batches by the application service
+        self._log.info("Deferring file processing to avoid memory exhaustion")
 
-        return self._create_scan_result(branches, commit_cache, tags, all_files)
-
-    async def _process_commits_concurrently(
-        self,
-        cloned_path: Path,
-        commits_batch: list[tuple[str, dict[str, Any]]],
-    ) -> dict[str, GitCommit]:
-        """Process a batch of commits concurrently."""
-        batch_cache = {}
-
-        async def process_single_commit(
-            commit_sha: str, commit_data: dict[str, Any]
-        ) -> tuple[str, GitCommit | None]:
-            git_commit = await self._create_git_commit_from_data(
-                cloned_path, commit_data
-            )
-            return commit_sha, git_commit
-
-        # Process commits concurrently in smaller batches
-        semaphore = asyncio.Semaphore(50)  # Limit concurrent operations
-
-        async def bounded_process(
-            item: tuple[str, dict[str, Any]],
-        ) -> tuple[str, GitCommit | None]:
-            async with semaphore:
-                return await process_single_commit(item[0], item[1])
-
-        # Process all commits concurrently
-        results = await asyncio.gather(
-            *[bounded_process(item) for item in commits_batch],
-            return_exceptions=True,
-        )
-
-        # Collect successful results
-        for result in results:
-            if isinstance(result, tuple):
-                # Type narrowing: result is now tuple[str, GitCommit | None]
-                commit_sha, git_commit = result
-                if git_commit is not None:
-                    batch_cache[commit_sha] = git_commit
-
-        return batch_cache
+        return self._create_scan_result(branches, commit_cache, tags, [], cloned_path)
 
     async def _process_branches_bulk(
         self,
@@ -167,30 +126,6 @@ class GitRepositoryScanner:
 
         return branches, commit_cache
 
-    async def _create_git_commit_from_data(
-        self, cloned_path: Path, commit_data: dict[str, Any], repo_id: int | None = None
-    ) -> GitCommit | None:
-        """Create GitCommit from pre-fetched commit data."""
-        commit_sha = commit_data["sha"]
-
-        # Get files for this commit
-        files_data = await self.git_adapter.get_commit_files(cloned_path, commit_sha)
-        self._create_git_files(cloned_path, files_data, commit_sha)
-        author = self._format_author_from_data(commit_data)
-
-        # Cache datetime creation
-        created_at = datetime.now(UTC)
-
-        return GitCommit(
-            created_at=created_at,
-            commit_sha=commit_sha,
-            repo_id=repo_id or 0,  # Use 0 as default if not provided
-            date=commit_data["date"],
-            message=commit_data["message"],
-            parent_commit_sha=commit_data["parent_sha"],
-            author=author,
-        )
-
     def _format_author_from_data(self, commit_data: dict[str, Any]) -> str:
         """Format author string from commit data."""
         author_name = commit_data.get("author_name", "")
@@ -376,17 +311,18 @@ class GitRepositoryScanner:
         branches: list[GitBranch],
         commit_cache: dict[str, GitCommit],
         tags: list[GitTag],
-        all_files: list[GitFile],
+        all_files: list[GitFile],  # noqa: ARG002
+        cloned_path: Path | None = None,  # noqa: ARG002
     ) -> RepositoryScanResult:
         """Create final scan result."""
-        # Files are loaded on-demand for performance, so total_files is 0 during scan
+        # Files list is empty to avoid memory issues - will be processed in batches
         scan_result = RepositoryScanResult(
             branches=branches,
             all_commits=list(commit_cache.values()),
             scan_timestamp=datetime.now(UTC),
-            total_files_across_commits=len(all_files),
+            total_files_across_commits=0,  # Will be updated after batch processing
             all_tags=tags,
-            all_files=all_files,
+            all_files=[],  # Empty - processed in batches to avoid memory exhaustion
         )
 
         self._log.info(
@@ -395,16 +331,35 @@ class GitRepositoryScanner:
         )
         return scan_result
 
-    async def _process_files(
-        self, cloned_path: Path, commit_cache: dict[str, GitCommit]
+    async def process_files_for_commits_batch(
+        self, cloned_path: Path, commit_shas: list[str]
     ) -> list[GitFile]:
-        """Process files for a commit."""
+        """Process files for a batch of commits.
+
+        This allows the application service to process files in batches
+        to avoid loading millions of files into memory at once.
+
+        CRITICAL: Reuses a single Repo object to avoid creating 32K+ Repo instances
+        which would consume massive memory (1-2 MB each).
+        """
+        from git import Repo
+
+        # Open repo once and reuse for all commits in this batch
+        repo = Repo(cloned_path)
         files = []
-        for commit_sha in commit_cache:
-            files_data = await self.git_adapter.get_commit_files(
-                cloned_path, commit_sha
-            )
-            files.extend(self._create_git_files(cloned_path, files_data, commit_sha))
+
+        try:
+            for commit_sha in commit_shas:
+                files_data = await self.git_adapter.get_commit_files(
+                    cloned_path, commit_sha, repo=repo
+                )
+                files.extend(
+                    self._create_git_files(cloned_path, files_data, commit_sha)
+                )
+        finally:
+            # Explicitly close the repo to free resources
+            repo.close()
+
         return files
 
 

{kodit-0.5.6 → kodit-0.5.7}/src/kodit/infrastructure/cloning/git/git_python_adaptor.py

@@ -346,14 +346,22 @@ class GitPythonAdapter(GitAdapter):
         )
 
     async def get_commit_files(
-        self, local_path: Path, commit_sha: str
+        self, local_path: Path, commit_sha: str, repo: Repo
     ) -> list[dict[str, Any]]:
-        """Get all files in a specific commit from the git tree."""
+        """Get all files in a specific commit from the git tree.
+
+        Args:
+            local_path: Path to the repository
+            commit_sha: SHA of the commit to get files for
+            repo: Repo object to reuse (avoids creating new Repo per commit)
+
+        """
 
         def _get_files() -> list[dict[str, Any]]:
             try:
-                repo = Repo(local_path)
-                commit = repo.commit(commit_sha)
+                # Use the provided repo object
+                _repo = repo
+                commit = _repo.commit(commit_sha)
 
                 files = []
 
@@ -395,7 +403,11 @@ class GitPythonAdapter(GitAdapter):
         """Get file metadata for a commit, with files checked out to disk."""
         await self._checkout_commit(local_path, commit_sha)
         try:
-            return await self.get_commit_files(local_path, commit_sha)
+            repo = Repo(local_path)
+            try:
+                return await self.get_commit_files(local_path, commit_sha, repo)
+            finally:
+                repo.close()
         finally:
             await self.restore_to_branch(local_path, "main")
 
@@ -570,6 +582,4 @@ class GitPythonAdapter(GitAdapter):
             else:
                 return diff_text
 
-        return await asyncio.get_event_loop().run_in_executor(
-            self.executor, _get_diff
-        )
+        return await asyncio.get_event_loop().run_in_executor(self.executor, _get_diff)

{kodit-0.5.6 → kodit-0.5.7}/tests/kodit/domain/services/git_repository_service_test.py

@@ -124,7 +124,9 @@ async def test_git_repository_scanner_scan_repository(
     assert len(result.branches) == 2
     assert len(result.all_commits) == 2
     assert len(result.all_tags) == 1
-    assert result.total_files_across_commits == 2
+    # Files are processed in batches by application service to avoid memory issues
+    assert result.total_files_across_commits == 0
+    assert len(result.all_files) == 0
 
     # Verify branches
     branch_names = {branch.name for branch in result.branches}

{kodit-0.5.6 → kodit-0.5.7}/tests/kodit/infrastructure/cloning/git/git_python_adaptor_test.py

@@ -128,9 +128,7 @@ async def test_pull_repository_failure(git_adapter: GitPythonAdapter) -> None:
     local_path = Path("/tmp/test-repo")
 
     with patch(f"{MODULE_PATH}.Repo") as mock_repo_class:
-        mock_repo_class.side_effect = InvalidGitRepositoryError(
-            "Not a git repository"
-        )
+        mock_repo_class.side_effect = InvalidGitRepositoryError("Not a git repository")
 
         with pytest.raises(InvalidGitRepositoryError):
             await git_adapter.pull_repository(local_path)
@@ -322,7 +320,7 @@ async def test_get_commit_files(git_adapter: GitPythonAdapter) -> None:
         mock_repo.commit.return_value = mock_commit
         mock_repo_class.return_value = mock_repo
 
-        result = await git_adapter.get_commit_files(local_path, commit_sha)
+        result = await git_adapter.get_commit_files(local_path, commit_sha, mock_repo)
 
         # Should return empty list since no blobs
         assert result == []
@@ -348,9 +346,7 @@ async def test_repository_exists_false(git_adapter: GitPythonAdapter) -> None:
     local_path = Path("/tmp/test-repo")
 
     with patch(f"{MODULE_PATH}.Repo") as mock_repo_class:
-        mock_repo_class.side_effect = InvalidGitRepositoryError(
-            "Not a git repository"
-        )
+        mock_repo_class.side_effect = InvalidGitRepositoryError("Not a git repository")
 
         result = await git_adapter.repository_exists(local_path)