kodit 0.5.5__tar.gz → 0.5.7__tar.gz

{kodit-0.5.5 → kodit-0.5.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.5.5
+Version: 0.5.7
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/

{kodit-0.5.5 → kodit-0.5.7}/docs/reference/api/index.md

@@ -12,7 +12,7 @@ look at the [hosted version](https://kodit.helix.ml/docs).
 This is the REST API for the Kodit server. Please refer to the
 [Kodit documentation](https://docs.helix.ml/kodit/) for more information.
     
-Current version: 0.5.5
+Current version: 0.5.6
 
 ## Authentication
 

{kodit-0.5.5 → kodit-0.5.7}/docs/reference/api/openapi.json

@@ -3,7 +3,7 @@
   "info": {
     "title": "kodit API",
     "description": "\nThis is the REST API for the Kodit server. Please refer to the\n[Kodit documentation](https://docs.helix.ml/kodit/) for more information.\n    ",
-    "version": "0.5.5"
+    "version": "0.5.6"
   },
   "paths": {
     "/healthz": {
@@ -2496,7 +2496,9 @@
           "kodit.commit.create_summary_enrichment",
           "kodit.commit.create_summary_embeddings",
           "kodit.commit.create_architecture_enrichment",
-          "kodit.commit.create_public_api_docs"
+          "kodit.commit.create_public_api_docs",
+          "kodit.commit.create_commit_description",
+          "kodit.commit.create_database_schema"
         ],
         "title": "TaskOperation",
         "description": "Task operation."

{kodit-0.5.5 → kodit-0.5.7}/src/kodit/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.5.5'
-__version_tuple__ = version_tuple = (0, 5, 5)
+__version__ = version = '0.5.7'
+__version_tuple__ = version_tuple = (0, 5, 7)
 
 __commit_id__ = commit_id = None

{kodit-0.5.5 → kodit-0.5.7}/src/kodit/app.py

@@ -63,6 +63,8 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[AppLifespanState]:
                 )
             )
         )
+    except StopAsyncIteration:
+        pass
     except Exception as e:
         raise ValueError("Embedding service is not accessible") from e
     try:

{kodit-0.5.5 → kodit-0.5.7}/src/kodit/application/factories/server_factory.py

@@ -50,6 +50,9 @@ from kodit.infrastructure.bm25.vectorchord_bm25_repository import (
     VectorChordBM25Repository,
 )
 from kodit.infrastructure.cloning.git.git_python_adaptor import GitPythonAdapter
+from kodit.infrastructure.database_schema.database_schema_detector import (
+    DatabaseSchemaDetector,
+)
 from kodit.infrastructure.embedding.embedding_factory import (
     embedding_domain_service_factory,
 )
@@ -255,6 +258,7 @@ class ServerFactory:
                 text_search_service=self.text_search_service(),
                 embedding_repository=self.embedding_repository(),
                 architecture_service=self.architecture_service(),
+                database_schema_detector=DatabaseSchemaDetector(),
                 enrichment_v2_repository=self.enrichment_v2_repository(),
                 enricher_service=self.enricher(),
                 enrichment_association_repository=self.enrichment_association_repository(),

{kodit-0.5.5 → kodit-0.5.7}/src/kodit/application/services/commit_indexing_application_service.py

@@ -14,6 +14,9 @@ if TYPE_CHECKING:
     from kodit.application.services.enrichment_query_service import (
         EnrichmentQueryService,
     )
+from kodit.domain.enrichments.architecture.database_schema.database_schema import (
+    DatabaseSchemaEnrichment,
+)
 from kodit.domain.enrichments.architecture.physical.physical import (
     PhysicalArchitectureEnrichment,
 )
@@ -27,11 +30,20 @@ from kodit.domain.enrichments.enrichment import (
     EnrichmentAssociation,
     EnrichmentV2,
 )
+from kodit.domain.enrichments.history.commit_description.commit_description import (
+    CommitDescriptionEnrichment,
+)
 from kodit.domain.enrichments.request import (
     EnrichmentRequest as GenericEnrichmentRequest,
 )
 from kodit.domain.entities import Task
-from kodit.domain.entities.git import GitFile, GitRepo, SnippetV2, TrackingType
+from kodit.domain.entities.git import (
+    GitCommit,
+    GitFile,
+    GitRepo,
+    SnippetV2,
+    TrackingType,
+)
 from kodit.domain.factories.git_repo_factory import GitRepoFactory
 from kodit.domain.protocols import (
     EnrichmentAssociationRepository,
@@ -63,6 +75,9 @@ from kodit.domain.value_objects import (
     TaskOperation,
     TrackableType,
 )
+from kodit.infrastructure.database_schema.database_schema_detector import (
+    DatabaseSchemaDetector,
+)
 from kodit.infrastructure.slicing.api_doc_extractor import APIDocExtractor
 from kodit.infrastructure.slicing.slicer import Slicer
 from kodit.infrastructure.sqlalchemy import entities as db_entities
@@ -82,6 +97,73 @@ You are a professional software developer. You will be given a snippet of code.
 Please provide a concise explanation of the code.
 """
 
+COMMIT_DESCRIPTION_SYSTEM_PROMPT = """
+You are a professional software developer. You will be given a git commit diff.
+Please provide a concise description of what changes were made and why.
+"""
+
+DATABASE_SCHEMA_SYSTEM_PROMPT = """
+You are an expert database architect and documentation specialist.
+Your task is to create clear, visual documentation of database schemas.
+"""
+
+DATABASE_SCHEMA_TASK_PROMPT = """
+You will be provided with a database schema discovery report.
+Please create comprehensive database schema documentation.
+
+<schema_report>
+{schema_report}
+</schema_report>
+
+**Return the following:**
+
+## Entity List
+
+For each table/entity, write one line:
+- **[Table Name]**: [brief description of what it stores]
+
+## Mermaid ERD
+
+Create a Mermaid Entity Relationship Diagram showing:
+- All entities (tables)
+- Key relationships between entities (if apparent from names or common patterns)
+- Use standard ERD notation
+
+Example format:
+```mermaid
+erDiagram
+    User ||--o{{ Order : places
+    User {{
+        int id PK
+        string email
+        string name
+    }}
+    Order {{
+        int id PK
+        int user_id FK
+        datetime created_at
+    }}
+```
+
+If specific field details aren't available, show just the entity boxes and
+relationships.
+
+## Key Observations
+
+Answer these questions in 1-2 sentences each:
+1. What is the primary data model pattern (e.g., user-centric,
+   event-sourced, multi-tenant)?
+2. What migration strategy is being used?
+3. Are there any notable database design patterns or concerns?
+
+## Rules:
+- Be concise and focus on the high-level structure
+- Infer reasonable relationships from table names when explicit information
+  isn't available
+- If no database schema is found, state that clearly
+- Keep entity descriptions to 10 words or less
+"""
+
 
 class CommitIndexingApplicationService:
     """Application service for commit indexing operations."""
@@ -103,6 +185,7 @@ class CommitIndexingApplicationService:
         text_search_service: EmbeddingDomainService,
         embedding_repository: SqlAlchemyEmbeddingRepository,
         architecture_service: PhysicalArchitectureService,
+        database_schema_detector: DatabaseSchemaDetector,
         enricher_service: Enricher,
         enrichment_v2_repository: EnrichmentV2Repository,
         enrichment_association_repository: EnrichmentAssociationRepository,
@@ -124,6 +207,7 @@ class CommitIndexingApplicationService:
         self.text_search_service = text_search_service
         self.embedding_repository = embedding_repository
         self.architecture_service = architecture_service
+        self.database_schema_detector = database_schema_detector
         self.enrichment_v2_repository = enrichment_v2_repository
         self.enrichment_association_repository = enrichment_association_repository
         self.enricher_service = enricher_service
@@ -191,11 +275,66 @@ class CommitIndexingApplicationService:
                 await self.process_architecture_discovery(repository_id, commit_sha)
             elif task.type == TaskOperation.CREATE_PUBLIC_API_DOCS_FOR_COMMIT:
                 await self.process_api_docs(repository_id, commit_sha)
+            elif task.type == TaskOperation.CREATE_COMMIT_DESCRIPTION_FOR_COMMIT:
+                await self.process_commit_description(repository_id, commit_sha)
+            elif task.type == TaskOperation.CREATE_DATABASE_SCHEMA_FOR_COMMIT:
+                await self.process_database_schema(repository_id, commit_sha)
             else:
                 raise ValueError(f"Unknown task type: {task.type}")
         else:
             raise ValueError(f"Unknown task type: {task.type}")
 
+    async def _process_files_in_batches(
+        self, cloned_path: Path, all_commits: list[GitCommit], batch_size: int = 100
+    ) -> int:
+        """Process file metadata for all commits in batches to avoid memory exhaustion.
+
+        This loads file metadata (paths, sizes, blob SHAs) in batches and saves them
+        incrementally to avoid holding millions of file objects in memory.
+
+        Args:
+            cloned_path: Path to the cloned repository
+            all_commits: List of all commits from scan
+            batch_size: Number of commits to process at once (default 100)
+
+        Returns:
+            Total number of files processed
+
+        """
+        total_files = 0
+        commit_shas = [commit.commit_sha for commit in all_commits]
+        total_batches = (len(commit_shas) + batch_size - 1) // batch_size
+
+        self._log.info(
+            f"Processing files for {len(commit_shas)} commits "
+            f"in {total_batches} batches"
+        )
+
+        # Process commits in batches
+        for i in range(0, len(commit_shas), batch_size):
+            batch = commit_shas[i : i + batch_size]
+            batch_num = i // batch_size + 1
+
+            self._log.debug(
+                f"Processing batch {batch_num}/{total_batches} ({len(batch)} commits)"
+            )
+
+            # Get file metadata for this batch of commits
+            files = await self.scanner.process_files_for_commits_batch(
+                cloned_path, batch
+            )
+
+            # Save file metadata to database immediately
+            if files:
+                await self.git_file_repository.save_bulk(files)
+                total_files += len(files)
+                self._log.debug(
+                    f"Batch {batch_num}: Saved {len(files)} files "
+                    f"(total so far: {total_files})"
+                )
+
+        return total_files
+
     async def process_clone_repo(self, repository_id: int) -> None:
         """Clone a repository."""
         async with self.operation.create_child(
@@ -233,8 +372,11 @@ class CommitIndexingApplicationService:
             await step.set_current(2, "Saving commits")
             await self.git_commit_repository.save_bulk(scan_result.all_commits)
 
-            await step.set_current(3, "Saving files")
-            await self.git_file_repository.save_bulk(scan_result.all_files)
+            await step.set_current(3, "Processing and saving files in batches")
+            total_files = await self._process_files_in_batches(
+                repo.cloned_path, scan_result.all_commits
+            )
+            self._log.info(f"Processed and saved {total_files} total files")
 
             await step.set_current(4, "Saving branches")
             if scan_result.branches:
@@ -798,6 +940,137 @@ class CommitIndexingApplicationService:
                     ]
                 )
 
+    async def process_commit_description(
+        self, repository_id: int, commit_sha: str
+    ) -> None:
+        """Handle COMMIT_DESCRIPTION task - generate commit descriptions."""
+        async with self.operation.create_child(
+            TaskOperation.CREATE_COMMIT_DESCRIPTION_FOR_COMMIT,
+            trackable_type=TrackableType.KODIT_REPOSITORY,
+            trackable_id=repository_id,
+        ) as step:
+            # Check if commit description already exists for this commit
+            if await self.enrichment_query_service.has_commit_description_for_commit(
+                commit_sha
+            ):
+                await step.skip("Commit description already exists for commit")
+                return
+
+            # Get repository path
+            repo = await self.repo_repository.get(repository_id)
+            if not repo.cloned_path:
+                raise ValueError(f"Repository {repository_id} has never been cloned")
+
+            await step.set_total(3)
+            await step.set_current(1, "Getting commit diff")
+
+            # Get the diff for this commit
+            diff = await self.scanner.git_adapter.get_commit_diff(
+                repo.cloned_path, commit_sha
+            )
+
+            if not diff or len(diff.strip()) == 0:
+                await step.skip("No diff found for commit")
+                return
+
+            await step.set_current(2, "Enriching commit description with LLM")
+
+            # Enrich the diff through the enricher
+            enrichment_request = GenericEnrichmentRequest(
+                id=commit_sha,
+                text=diff,
+                system_prompt=COMMIT_DESCRIPTION_SYSTEM_PROMPT,
+            )
+
+            enriched_content = ""
+            async for response in self.enricher_service.enrich([enrichment_request]):
+                enriched_content = response.text
+
+            # Create and save commit description enrichment
+            enrichment = await self.enrichment_v2_repository.save(
+                CommitDescriptionEnrichment(
+                    content=enriched_content,
+                )
+            )
+            if not enrichment or not enrichment.id:
+                raise ValueError(
+                    f"Failed to save commit description enrichment for commit "
+                    f"{commit_sha}"
+                )
+            await self.enrichment_association_repository.save(
+                CommitEnrichmentAssociation(
+                    enrichment_id=enrichment.id,
+                    entity_id=commit_sha,
+                )
+            )
+
+            await step.set_current(3, "Commit description enrichment completed")
+
+    async def process_database_schema(
+        self, repository_id: int, commit_sha: str
+    ) -> None:
+        """Handle DATABASE_SCHEMA task - discover and document database schemas."""
+        async with self.operation.create_child(
+            TaskOperation.CREATE_DATABASE_SCHEMA_FOR_COMMIT,
+            trackable_type=TrackableType.KODIT_REPOSITORY,
+            trackable_id=repository_id,
+        ) as step:
+            # Check if database schema already exists for this commit
+            if await self.enrichment_query_service.has_database_schema_for_commit(
+                commit_sha
+            ):
+                await step.skip("Database schema already exists for commit")
+                return
+
+            # Get repository path
+            repo = await self.repo_repository.get(repository_id)
+            if not repo.cloned_path:
+                raise ValueError(f"Repository {repository_id} has never been cloned")
+
+            await step.set_total(3)
+            await step.set_current(1, "Discovering database schemas")
+
+            # Discover database schemas
+            schema_report = await self.database_schema_detector.discover_schemas(
+                repo.cloned_path
+            )
+
+            if "No database schemas detected" in schema_report:
+                await step.skip("No database schemas found in repository")
+                return
+
+            await step.set_current(2, "Enriching schema documentation with LLM")
+
+            # Enrich the schema report through the enricher
+            enrichment_request = GenericEnrichmentRequest(
+                id=commit_sha,
+                text=DATABASE_SCHEMA_TASK_PROMPT.format(schema_report=schema_report),
+                system_prompt=DATABASE_SCHEMA_SYSTEM_PROMPT,
+            )
+
+            enriched_content = ""
+            async for response in self.enricher_service.enrich([enrichment_request]):
+                enriched_content = response.text
+
+            # Create and save database schema enrichment
+            enrichment = await self.enrichment_v2_repository.save(
+                DatabaseSchemaEnrichment(
+                    content=enriched_content,
+                )
+            )
+            if not enrichment or not enrichment.id:
+                raise ValueError(
+                    f"Failed to save database schema enrichment for commit {commit_sha}"
+                )
+            await self.enrichment_association_repository.save(
+                CommitEnrichmentAssociation(
+                    enrichment_id=enrichment.id,
+                    entity_id=commit_sha,
+                )
+            )
+
+            await step.set_current(3, "Database schema enrichment completed")
+
     async def _new_snippets_for_type(
         self, all_snippets: list[EnrichmentV2], embedding_type: EmbeddingType
     ) -> list[EnrichmentV2]:

{kodit-0.5.5 → kodit-0.5.7}/src/kodit/application/services/enrichment_query_service.py

@@ -5,6 +5,9 @@ import structlog
 from kodit.domain.enrichments.architecture.architecture import (
     ENRICHMENT_TYPE_ARCHITECTURE,
 )
+from kodit.domain.enrichments.architecture.database_schema.database_schema import (
+    ENRICHMENT_SUBTYPE_DATABASE_SCHEMA,
+)
 from kodit.domain.enrichments.architecture.physical.physical import (
     ENRICHMENT_SUBTYPE_PHYSICAL,
 )
@@ -14,6 +17,10 @@ from kodit.domain.enrichments.development.snippet.snippet import (
     ENRICHMENT_SUBTYPE_SNIPPET_SUMMARY,
 )
 from kodit.domain.enrichments.enrichment import EnrichmentAssociation, EnrichmentV2
+from kodit.domain.enrichments.history.commit_description.commit_description import (
+    ENRICHMENT_SUBTYPE_COMMIT_DESCRIPTION,
+)
+from kodit.domain.enrichments.history.history import ENRICHMENT_TYPE_HISTORY
 from kodit.domain.enrichments.usage.api_docs import ENRICHMENT_SUBTYPE_API_DOCS
 from kodit.domain.enrichments.usage.usage import ENRICHMENT_TYPE_USAGE
 from kodit.domain.protocols import (
@@ -215,6 +222,36 @@ class EnrichmentQueryService:
         api_docs = await self.get_api_docs_for_commit(commit_sha)
         return len(api_docs) > 0
 
+    async def get_commit_description_for_commit(
+        self, commit_sha: str
+    ) -> list[EnrichmentV2]:
+        """Get commit description enrichments for a commit."""
+        return await self.get_enrichments_for_commit(
+            commit_sha,
+            enrichment_type=ENRICHMENT_TYPE_HISTORY,
+            enrichment_subtype=ENRICHMENT_SUBTYPE_COMMIT_DESCRIPTION,
+        )
+
+    async def has_commit_description_for_commit(self, commit_sha: str) -> bool:
+        """Check if a commit has commit description enrichments."""
+        commit_descriptions = await self.get_commit_description_for_commit(commit_sha)
+        return len(commit_descriptions) > 0
+
+    async def get_database_schema_for_commit(
+        self, commit_sha: str
+    ) -> list[EnrichmentV2]:
+        """Get database schema enrichments for a commit."""
+        return await self.get_enrichments_for_commit(
+            commit_sha,
+            enrichment_type=ENRICHMENT_TYPE_ARCHITECTURE,
+            enrichment_subtype=ENRICHMENT_SUBTYPE_DATABASE_SCHEMA,
+        )
+
+    async def has_database_schema_for_commit(self, commit_sha: str) -> bool:
+        """Check if a commit has database schema enrichments."""
+        database_schemas = await self.get_database_schema_for_commit(commit_sha)
+        return len(database_schemas) > 0
+
     async def associations_for_enrichments(
         self, enrichments: list[EnrichmentV2]
     ) -> list[EnrichmentAssociation]:

kodit-0.5.7/src/kodit/domain/enrichments/architecture/database_schema/__init__.py

@@ -0,0 +1 @@
+"""Database schema enrichments."""

kodit-0.5.7/src/kodit/domain/enrichments/architecture/database_schema/database_schema.py

@@ -0,0 +1,17 @@
+"""Database schema enrichment domain entity."""
+
+from dataclasses import dataclass
+
+from kodit.domain.enrichments.architecture.architecture import ArchitectureEnrichment
+
+ENRICHMENT_SUBTYPE_DATABASE_SCHEMA = "database_schema"
+
+
+@dataclass(frozen=True)
+class DatabaseSchemaEnrichment(ArchitectureEnrichment):
+    """Enrichment containing database schema information for a commit."""
+
+    @property
+    def subtype(self) -> str | None:
+        """Return the enrichment subtype."""
+        return ENRICHMENT_SUBTYPE_DATABASE_SCHEMA

kodit-0.5.7/src/kodit/domain/enrichments/history/__init__.py

@@ -0,0 +1 @@
+"""History enrichments."""

kodit-0.5.7/src/kodit/domain/enrichments/history/commit_description/__init__.py

@@ -0,0 +1 @@
+"""Commit description enrichments."""

kodit-0.5.7/src/kodit/domain/enrichments/history/commit_description/commit_description.py

@@ -0,0 +1,17 @@
+"""Commit description enrichment domain entity."""
+
+from dataclasses import dataclass
+
+from kodit.domain.enrichments.history.history import HistoryEnrichment
+
+ENRICHMENT_SUBTYPE_COMMIT_DESCRIPTION = "commit_description"
+
+
+@dataclass(frozen=True)
+class CommitDescriptionEnrichment(HistoryEnrichment):
+    """Enrichment containing a description of what a commit did."""
+
+    @property
+    def subtype(self) -> str | None:
+        """Return the enrichment subtype."""
+        return ENRICHMENT_SUBTYPE_COMMIT_DESCRIPTION

kodit-0.5.7/src/kodit/domain/enrichments/history/history.py

@@ -0,0 +1,18 @@
+"""History enrichment domain entity."""
+
+from abc import ABC
+from dataclasses import dataclass
+
+from kodit.domain.enrichments.enrichment import CommitEnrichment
+
+ENRICHMENT_TYPE_HISTORY = "history"
+
+
+@dataclass(frozen=True)
+class HistoryEnrichment(CommitEnrichment, ABC):
+    """Enrichment containing historical information for a commit."""
+
+    @property
+    def type(self) -> str:
+        """Return the enrichment type."""
+        return ENRICHMENT_TYPE_HISTORY

{kodit-0.5.5 → kodit-0.5.7}/src/kodit/domain/protocols.py

@@ -4,6 +4,8 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Protocol, TypeVar
 
+from git import Repo
+
 from kodit.domain.enrichments.enrichment import EnrichmentAssociation, EnrichmentV2
 from kodit.domain.entities import (
     Task,
@@ -163,9 +165,16 @@ class GitAdapter(ABC):
 
     @abstractmethod
     async def get_commit_files(
-        self, local_path: Path, commit_sha: str
+        self, local_path: Path, commit_sha: str, repo: Repo
     ) -> list[dict[str, Any]]:
-        """Get all files in a specific commit from the git tree."""
+        """Get all files in a specific commit from the git tree.
+
+        Args:
+            local_path: Path to the repository
+            commit_sha: SHA of the commit to get files for
+            repo: Repo object to reuse (avoids creating new Repo per commit)
+
+        """
 
     @abstractmethod
     async def get_commit_file_data(
@@ -213,6 +222,10 @@ class GitAdapter(ABC):
     ) -> list[str]:
         """Get only commit SHAs for a branch (much faster than full commit data)."""
 
+    @abstractmethod
+    async def get_commit_diff(self, local_path: Path, commit_sha: str) -> str:
+        """Get the diff for a specific commit."""
+
 
 class SnippetRepositoryV2(ABC):
     """Repository for snippet operations."""