kodit 0.2.5__tar.gz → 0.2.7__tar.gz

{kodit-0.2.5 → kodit-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.2.5
+Version: 0.2.7
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/

{kodit-0.2.5 → kodit-0.2.7}/docs/reference/configuration/index.md

@@ -1,7 +1,7 @@
 ---
 title: Configuration
 description: How to configure Kodit.
-weight: 1
+weight: 10
 ---
 
 Configuration of Kodit is performed by setting environmental variables or adding

{kodit-0.2.5 → kodit-0.2.7}/docs/reference/deployment/index.md

@@ -1,7 +1,7 @@
 ---
 title: Deployment
 description: Deploying Kodit with Docker Compose and Kubernetes.
-weight: 10
+weight: 20
 ---
 
 Kodit is packaged as a Docker container so you can run it on any popular orchestration platform. This page describes how to deploy Kodit as a service.
@@ -32,4 +32,3 @@ Deploy with `kubectl -n kodit apply -f kubernetes.yaml`
 
 1. `kind create cluster`
 2. `kubectl -n kodit apply -f kubernetes.yaml`
-

kodit-0.2.7/docs/reference/indexing/index.md

@@ -0,0 +1,279 @@
+---
+title: Indexing
+description: Learn how to index code sources in Kodit for AI-powered code search and generation.
+weight: 1
+---
+
+Kodit's indexing system allows you to create searchable indexes of your codebases, enabling AI assistants to find and reference relevant code snippets. This page explains how indexing works, what sources are supported, and how to use the indexing features.
+
+## How Indexing Works
+
+Kodit's indexing process consists of several stages:
+
+1. **Source Creation**: Kodit clones or copies your source code to a local working directory
+2. **File Processing**: Files are scanned and metadata is extracted (timestamps, authors, etc.)
+3. **Snippet Extraction**: Code is parsed using tree-sitter to extract meaningful snippets (functions, classes, methods)
+4. **Index Building**: Multiple search indexes are created:
+   - **BM25 Index**: For keyword-based search
+   - **Semantic Code Index**: For code similarity search using embeddings
+5. **Enrichment**: AI-powered enrichment of snippets for better search results
+   - **Semantic Text Index**: For natural language search using embeddings
+
+### Supported Source Types
+
+Kodit supports two main types of sources:
+
+#### Git Repositories
+
+Kodit can index any Git repository accessible via standard Git protocols:
+
+- **HTTPS**: Public repositories and private repositories with authentication
+- **SSH**: Using SSH keys for authentication
+- **Git Protocol**: For public repositories
+
+#### Local Directories
+
+Kodit can index local directories on your filesystem:
+
+- **Absolute paths**: `/path/to/your/code`
+- **Relative paths**: `./my-project`
+- **Home directory expansion**: `~/projects/my-app`
+- **File URIs**: `file:///path/to/your/code`
+
+## Basic Usage
+
+### Indexing a Source
+
+To index a source, use the `kodit index` command followed by the source location:
+
+```sh
+# Index a local directory
+kodit index /path/to/your/code
+
+# Index a public Git repository
+kodit index https://github.com/pydantic/pydantic
+
+# Index a private Git repository (requires authentication)
+kodit index https://github.com/username/private-repo
+
+# Index using SSH
+kodit index git@github.com:username/repo.git
+```
+
+### Listing Indexes
+
+To see all your indexed sources:
+
+```sh
+kodit index
+```
+
+This will display a table showing:
+
+- Index ID
+- Creation and update timestamps
+- Source URI
+- Number of snippets extracted
+
+## Git Protocol Support
+
+### HTTPS Authentication
+
+For private repositories, you can authenticate using:
+
+1. **Personal Access Token** (GitHub, GitLab, etc.):
+
+   ```sh
+   kodit index https://username:token@github.com/username/repo.git
+   ```
+
+2. **Username/Password** (if supported by your Git provider):
+
+   ```sh
+   kodit index https://username:password@github.com/username/repo.git
+   ```
+
+### SSH Authentication
+
+For SSH-based repositories:
+
+1. **SSH Key Authentication**:
+
+   ```sh
+   kodit index git@github.com:username/repo.git
+   ```
+
+   Ensure your SSH key is properly configured in your SSH agent or `~/.ssh/config`.
+
+2. **SSH with Custom Port**:
+
+   ```sh
+   kodit index ssh://git@github.com:2222/username/repo.git
+   ```
+
+### Git Providers
+
+Kodit works with any Git provider that supports standard Git protocols:
+
+- **GitHub**: `https://github.com/username/repo.git`
+- **GitLab**: `https://gitlab.com/username/repo.git`
+- **Bitbucket**: `https://bitbucket.org/username/repo.git`
+- **Azure DevOps**: `https://dev.azure.com/organization/project/_git/repo`
+- **Self-hosted Git servers**: Any Git server supporting HTTP/HTTPS or SSH
+
+## Examples of Use
+
+### Index a Public Azure DevOps Repository
+
+```sh
+kodit index https://winderai@dev.azure.com/winderai/public-test/_git/simple-ddd-brewing-demo
+```
+
+### Indexing a Private Azure DevOps Repository
+
+If you're accessing Azure DevOps from your local machine and have the Git credential
+helper you should be able to clone the repository as usual (obviously you won't be able
+to clone this because it is private):
+
+```sh
+kodit index https://winderai@dev.azure.com/winderai/private-test/_git/private-test
+```
+
+You can also use a Personal Access Token (PAT):
+
+```sh
+kodit index https://phil:xxxxxxSECRET_PATxxxxxxx@dev.azure.com/winderai/private-test/_git/private-test
+```
+
+## File Processing and Filtering
+
+### Ignored Files
+
+Kodit respects [standard ignore patterns](#ignore-patterns):
+
+- **`.gitignore`**: Standard Git ignore patterns
+- **`.noindex`**: Custom ignore patterns for Kodit (uses gitignore syntax)
+
+### Supported File Types
+
+Kodit automatically detects and processes files based on their extensions:
+
+| Language | Extensions |
+|----------|------------|
+| Python | `.py` |
+| JavaScript | `.js`, `.jsx` |
+| TypeScript | `.ts`, `.tsx` |
+| Go | `.go` |
+| C# | `.cs` |
+
+### Snippet Extraction
+
+Kodit uses tree-sitter to intelligently extract code snippets:
+
+- **Functions and Methods**: Complete function definitions with their bodies
+- **Classes**: Class definitions and their methods
+- **Imports**: Import statements for context
+- **Dependencies**: Ancestor classes and functions that the snippet depends on
+
+## Configuration
+
+### Clone Directory
+
+By default, Kodit stores cloned repositories in `~/.kodit/clones/`. You can configure this using the `DATA_DIR` environment variable:
+
+```sh
+export DATA_DIR=/custom/path/to/kodit/data
+```
+
+### Database Configuration
+
+Kodit uses SQLite by default, but supports PostgreSQL with VectorChord for better performance:
+
+```sh
+# SQLite (default)
+DB_URL=sqlite+aiosqlite:///path/to/kodit.db
+
+# PostgreSQL with VectorChord
+DB_URL=postgresql+asyncpg://user:password@localhost:5432/kodit
+DEFAULT_SEARCH_PROVIDER=vectorchord
+```
+
+### AI Provider Configuration
+
+For semantic search and enrichment, configure your AI provider:
+
+```sh
+# OpenAI
+DEFAULT_ENDPOINT_TYPE=openai
+DEFAULT_ENDPOINT_BASE_URL=https://api.openai.com/v1
+DEFAULT_ENDPOINT_API_KEY=sk-your-api-key
+
+# Or use local models (slower but private)
+# No configuration needed - uses local models by default
+```
+
+## Advanced Features
+
+### Re-indexing Sources
+
+Future feature!
+
+### Progress Monitoring
+
+Kodit shows progress during indexing operations:
+
+- File processing progress
+- Snippet extraction progress
+- Index building progress (BM25, embeddings)
+
+### Error Handling
+
+Common issues and solutions:
+
+1. **Authentication Errors**: Ensure your credentials are correct
+2. **Network Issues**: Check your internet connection and firewall settings
+3. **Permission Errors**: Ensure you have read access to the source
+4. **Unsupported Files**: Kodit will skip unsupported file types automatically
+
+## Privacy and Security
+
+### Local Processing
+
+- All code is processed locally by default
+- No code is sent to external services unless you configure AI providers
+- Cloned repositories are stored locally in your data directory
+
+### Ignore Patterns
+
+Kodit respects privacy by honoring:
+
+- `.gitignore` patterns
+- `.noindex` files for custom exclusions
+- Hidden files and directories (starting with `.`)
+
+### Authentication
+
+- SSH keys and tokens are handled by your system's Git configuration
+- Kodit doesn't store or transmit credentials
+- Use environment variables for sensitive configuration
+
+## Troubleshooting
+
+### Common Issues
+
+1. **"Failed to clone repository"**: Check your Git credentials and network connection
+2. **"Unsupported source"**: Ensure the path or URL is valid and accessible
+3. **"No snippets found"**: Check if the source contains supported file types
+4. **"Permission denied"**: Ensure you have read access to the source
+
+### Checking Index Status
+
+To verify your indexes are working correctly:
+
+```sh
+# List all indexes
+kodit index
+
+# Test search functionality
+kodit search text "example function"
+```

{kodit-0.2.5 → kodit-0.2.7}/src/kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.5'
-__version_tuple__ = version_tuple = (0, 2, 5)
+__version__ = version = '0.2.7'
+__version_tuple__ = version_tuple = (0, 2, 7)

{kodit-0.2.5 → kodit-0.2.7}/src/kodit/application/services/indexing_application_service.py

@@ -1,6 +1,7 @@
 """Application service for indexing operations."""
 
 import structlog
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.application.commands.snippet_commands import CreateIndexSnippetsCommand
 from kodit.application.services.snippet_application_service import (
@@ -8,6 +9,7 @@ from kodit.application.services.snippet_application_service import (
 )
 from kodit.domain.entities import Snippet
 from kodit.domain.enums import SnippetExtractionStrategy
+from kodit.domain.errors import EmptySourceError
 from kodit.domain.interfaces import ProgressCallback
 from kodit.domain.services.bm25_service import BM25DomainService
 from kodit.domain.services.embedding_service import EmbeddingDomainService
@@ -51,6 +53,7 @@ class IndexingApplicationService:
         text_search_service: EmbeddingDomainService,
         enrichment_service: EnrichmentDomainService,
         snippet_application_service: SnippetApplicationService,
+        session: AsyncSession,
     ) -> None:
         """Initialize the indexing application service.
 
@@ -62,11 +65,13 @@ class IndexingApplicationService:
             text_search_service: The text search domain service.
             enrichment_service: The enrichment domain service.
             snippet_application_service: The snippet application service.
+            session: The database session for transaction management.
 
         """
         self.indexing_domain_service = indexing_domain_service
         self.source_service = source_service
         self.snippet_application_service = snippet_application_service
+        self.session = session
         self.log = structlog.get_logger(__name__)
         self.bm25_service = bm25_service
         self.code_search_service = code_search_service
@@ -93,7 +98,12 @@ class IndexingApplicationService:
 
         # Create the index
         request = IndexCreateRequest(source_id=source.id)
-        return await self.indexing_domain_service.create_index(request)
+        index_view = await self.indexing_domain_service.create_index(request)
+
+        # Commit the index creation
+        await self.session.commit()
+
+        return index_view
 
     async def list_indexes(self) -> list[IndexView]:
         """List all available indexes with their details.
@@ -125,7 +135,7 @@ class IndexingApplicationService:
             progress_callback: Optional progress callback for reporting progress.
 
         Raises:
-            ValueError: If the index doesn't exist.
+            ValueError: If the index doesn't exist or no indexable snippets are found.
 
         """
         log_event("kodit.index.run")
@@ -138,8 +148,11 @@ class IndexingApplicationService:
 
         # Delete old snippets so we don't duplicate
         await self.indexing_domain_service.delete_all_snippets(index.id)
+        # Commit the deletion
+        await self.session.commit()
 
         # Create snippets for supported file types using the snippet application service
+        # (snippet_application_service handles its own commits)
         self.log.info("Creating snippets for files", index_id=index.id)
         command = CreateIndexSnippetsCommand(
             index_id=index.id, strategy=SnippetExtractionStrategy.METHOD_BASED
@@ -150,6 +163,11 @@ class IndexingApplicationService:
 
         snippets = await self.indexing_domain_service.get_snippets_for_index(index.id)
 
+        # Check if any snippets were extracted
+        if not snippets:
+            msg = f"No indexable snippets found for index {index.id}"
+            raise EmptySourceError(msg)
+
         # Create BM25 index
         self.log.info("Creating keyword index")
         reporter = Reporter(self.log, progress_callback)
@@ -184,6 +202,8 @@ class IndexingApplicationService:
 
         # Update index timestamp
         await self.indexing_domain_service.update_index_timestamp(index.id)
+        # Commit the timestamp update
+        await self.session.commit()
 
     async def _create_bm25_index(
         self, snippets: list[Snippet], progress_callback: ProgressCallback | None = None
@@ -264,6 +284,10 @@ class IndexingApplicationService:
                 "enrichment", processed, len(snippets), "Enriching snippets..."
             )
 
+        # Commit all snippet content updates as a single transaction
+        if enriched_contents:
+            await self.session.commit()
+
         await reporter.done("enrichment")
 
     async def _create_text_embeddings(

{kodit-0.2.5 → kodit-0.2.7}/src/kodit/application/services/snippet_application_service.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from typing import Any
 
 import structlog
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.application.commands.snippet_commands import (
     CreateIndexSnippetsCommand,
@@ -28,6 +29,7 @@ class SnippetApplicationService:
         snippet_extraction_service: SnippetExtractionDomainService,
         snippet_repository: SnippetRepository,
         file_repository: FileRepository,
+        session: AsyncSession,
     ) -> None:
         """Initialize the snippet application service.
 
@@ -35,11 +37,13 @@ class SnippetApplicationService:
             snippet_extraction_service: Domain service for snippet extraction
             snippet_repository: Repository for snippet persistence
             file_repository: Repository for file operations
+            session: The database session for transaction management
 
         """
         self.snippet_extraction_service = snippet_extraction_service
         self.snippet_repository = snippet_repository
         self.file_repository = file_repository
+        self.session = session
         self.log = structlog.get_logger(__name__)
 
     async def extract_snippets_from_file(
@@ -140,4 +144,6 @@ class SnippetApplicationService:
                 message=f"Processing {file.cloned_path}...",
             )
 
+        # Commit all snippet creations in a single transaction
+        await self.session.commit()
         await reporter.done("create_snippets")

{kodit-0.2.5 → kodit-0.2.7}/src/kodit/cli.py

@@ -19,6 +19,7 @@ from kodit.config import (
     with_app_context,
     with_session,
 )
+from kodit.domain.errors import EmptySourceError
 from kodit.domain.services.source_service import SourceService
 from kodit.domain.value_objects import MultiSearchRequest
 from kodit.infrastructure.indexing.indexing_factory import (
@@ -58,6 +59,7 @@ def create_snippet_application_service(
         snippet_extraction_service=snippet_extraction_service,
         snippet_repository=snippet_repository,
         file_repository=file_repository,
+        session=session,
     )
 
 
@@ -100,6 +102,7 @@ async def index(
     sources: list[str],
 ) -> None:
     """List indexes, or index data sources."""
+    log = structlog.get_logger(__name__)
     source_service = SourceService(
         clone_dir=app_context.get_clone_dir(),
         session_factory=lambda: session,
@@ -152,7 +155,17 @@ async def index(
 
         # Create a new progress callback for the indexing operations
         indexing_progress_callback = create_multi_stage_progress_callback()
-        await service.run_index(index.id, indexing_progress_callback)
+        try:
+            await service.run_index(index.id, indexing_progress_callback)
+        except EmptySourceError as e:
+            log.exception("Empty source error", error=e)
+            msg = f"""{e}. This could mean:
+• The repository contains no supported file types
+• All files are excluded by ignore patterns
+• The files contain no extractable code snippets
+Please check the repository contents and try again.
+"""
+            click.echo(msg)
 
 
 @cli.group()

kodit-0.2.7/src/kodit/domain/errors.py

@@ -0,0 +1,5 @@
+"""Domain errors."""
+
+
+class EmptySourceError(ValueError):
+    """Error raised when a source is empty."""

{kodit-0.2.5 → kodit-0.2.7}/src/kodit/domain/services/source_service.py

@@ -49,7 +49,7 @@ class SourceService:
         """Create a source."""
         async with self._session_factory() as session:
             repo = SqlAlchemySourceRepository(session)
-            git_factory, folder_factory = self._build_factories(repo)
+            git_factory, folder_factory = self._build_factories(repo, session)
 
             if is_valid_clone_target(uri_or_path_like):
                 source = await git_factory.create(uri_or_path_like, progress_callback)
@@ -60,17 +60,18 @@ class SourceService:
             else:
                 raise ValueError(f"Unsupported source: {uri_or_path_like}")
 
-            await session.commit()  # one commit for all work
+            # Factories handle their own commits now
             return source
 
     def _build_factories(
-        self, repository: SourceRepository
+        self, repository: SourceRepository, session: AsyncSession
     ) -> tuple[GitSourceFactory, FolderSourceFactory]:
         # Git-specific collaborators
         git_wc = GitWorkingCopyProvider(self.clone_dir)
         git_factory = GitSourceFactory(
             repository=repository,
             working_copy=git_wc,
+            session=session,
         )
 
         # Folder-specific collaborators
@@ -78,6 +79,7 @@ class SourceService:
         folder_factory = FolderSourceFactory(
             repository=repository,
             working_copy=fold_wc,
+            session=session,
         )
 
         return git_factory, folder_factory

{kodit-0.2.5 → kodit-0.2.7}/src/kodit/infrastructure/cloning/folder/factory.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 
 import structlog
+from sqlalchemy.ext.asyncio import AsyncSession
 
 from kodit.domain.entities import AuthorFileMapping, Source, SourceType
 from kodit.domain.interfaces import NullProgressCallback, ProgressCallback
@@ -22,6 +23,7 @@ class FolderSourceFactory:
         self,
         repository: SourceRepository,
         working_copy: FolderWorkingCopyProvider,
+        session: AsyncSession,
     ) -> None:
         """Initialize the source factory."""
         self.log = structlog.get_logger(__name__)
@@ -29,6 +31,7 @@ class FolderSourceFactory:
         self.working_copy = working_copy
         self.metadata_extractor = FolderFileMetadataExtractor()
         self.author_extractor = NoOpAuthorExtractor()
+        self.session = session
 
     async def create(
         self, uri: str, progress_callback: ProgressCallback | None = None
@@ -55,7 +58,7 @@ class FolderSourceFactory:
         clone_path = await self.working_copy.prepare(directory.as_uri())
 
         # Create source record
-        source = await self.repository.create_source(
+        source = await self.repository.save(
             Source(
                 uri=directory.as_uri(),
                 cloned_path=str(clone_path),
@@ -63,12 +66,18 @@ class FolderSourceFactory:
             )
         )
 
+        # Commit source creation so we get an ID for foreign key relationships
+        await self.session.commit()
+
         # Get all files to process
         files = [f for f in clone_path.rglob("*") if f.is_file()]
 
         # Process files
         await self._process_files(source, files, progress_callback)
 
+        # Commit file processing
+        await self.session.commit()
+
         return source
 
     async def _process_files(