kodit 0.2.9__tar.gz → 0.3.0__tar.gz

{kodit-0.2.9 → kodit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kodit
-Version: 0.2.9
+Version: 0.3.0
 Summary: Code indexing for better AI code generation
 Project-URL: Homepage, https://docs.helixml.tech/kodit/
 Project-URL: Documentation, https://docs.helixml.tech/kodit/

{kodit-0.2.9 → kodit-0.3.0}/docs/reference/deployment/docker-compose.yaml

@@ -29,6 +29,10 @@ services:
       ENRICHMENT_ENDPOINT_API_KEY: REPLACE_WITH_YOUR_API_KEY
       ENRICHMENT_ENDPOINT_MODEL: o3-mini
 
+      # Auto-indexing configuration
+      AUTO_INDEXING_SOURCES_0_URI: https://github.com/pydantic/pydantic
+      AUTO_INDEXING_SOURCES_1_URI: https://github.com/fastapi/fastapi
+      AUTO_INDEXING_SOURCES_2_URI: https://github.com/helix-ml/kodit
 
   vectorchord:
     image: tensorchord/vchord-suite:pg17-20250601

{kodit-0.2.9 → kodit-0.3.0}/docs/reference/deployment/kubernetes.yaml

@@ -77,6 +77,10 @@ spec:
               value: "REPLACE_WITH_YOUR_API_KEY"
             - name: ENRICHMENT_ENDPOINT_MODEL
               value: "o3-mini"
+            - name: AUTO_INDEXING_SOURCES_0_URI
+              value: "https://github.com/pydantic/pydantic"
+            - name: AUTO_INDEXING_SOURCES_1_URI
+              value: "https://github.com/helix-ml/kodit"
           ports:
             - containerPort: 8080
           readinessProbe:

{kodit-0.2.9 → kodit-0.3.0}/docs/reference/indexing/index.md

@@ -42,7 +42,7 @@ Kodit can index local directories on your filesystem:
 
 ## Basic Usage
 
-### Indexing a Source
+### Manual Indexing
 
 To index a source, use the `kodit index` command followed by the source location:
 
@@ -75,6 +75,61 @@ This will display a table showing:
 - Source URI
 - Number of snippets extracted
 
+### Auto-Indexing
+
+If you're running Kodit as a shared server, you need to configure what gets indexed.
+Auto-indexing is a simple indexing configuration powered by environmental variables.
+
+#### Configuration via Environment Variables
+
+Configure auto-indexing sources using environment variables with the `AUTO_INDEXING_SOURCES_{X}_` prefix:
+
+```sh
+# Configure a single auto-index source
+export AUTO_INDEXING_SOURCES_0_URI="https://github.com/pydantic/pydantic"
+
+# Configure multiple auto-index sources
+export AUTO_INDEXING_SOURCES_0_URI="https://github.com/pydantic/pydantic"
+export AUTO_INDEXING_SOURCES_1_URI="https://github.com/fastapi/fastapi"
+export AUTO_INDEXING_SOURCES_2_URI="/path/to/local/project"
+
+# Or use a .env file
+echo "AUTO_INDEXING_SOURCES_0_URI=https://github.com/pydantic/pydantic" >> .env
+echo "AUTO_INDEXING_SOURCES_1_URI=https://github.com/fastapi/fastapi" >> .env
+echo "AUTO_INDEXING_SOURCES_2_URI=/path/to/local/project" >> .env
+```
+
+**Configuration Format:**
+
+- Use `AUTO_INDEXING_SOURCES_N_URI` where `N` is a zero-based index
+- Sources are indexed in numerical order (0, 1, 2, etc.)
+- Supports all source types: Git repositories (HTTPS/SSH) and local directories
+- Gaps in numbering are allowed (e.g., 0, 2, 5 will work)
+
+#### Using Auto-Indexing
+
+To manually index all configured auto-index sources:
+
+```sh
+kodit index --auto-index
+```
+
+This command will:
+
+1. Read the auto-indexing configuration from environment variables
+2. Index each configured source in sequence
+3. Show progress for each source being indexed
+4. Handle errors gracefully and continue with remaining sources
+
+If no auto-index sources are configured, the command will display a message indicating
+that no sources are configured.
+
+To automatically run index all configured auto-index sources:
+
+```sh
+kodit serve
+```
+
 ## Git Protocol Support
 
 ### HTTPS Authentication
@@ -226,15 +281,6 @@ Kodit shows progress during indexing operations:
 - Snippet extraction progress
 - Index building progress (BM25, embeddings)
 
-### Error Handling
-
-Common issues and solutions:
-
-1. **Authentication Errors**: Ensure your credentials are correct
-2. **Network Issues**: Check your internet connection and firewall settings
-3. **Permission Errors**: Ensure you have read access to the source
-4. **Unsupported Files**: Kodit will skip unsupported file types automatically
-
 ## Privacy and Security
 
 ### Local Processing
@@ -265,6 +311,8 @@ Kodit respects privacy by honoring:
 2. **"Unsupported source"**: Ensure the path or URL is valid and accessible
 3. **"No snippets found"**: Check if the source contains supported file types
 4. **"Permission denied"**: Ensure you have read access to the source
+5. **"No auto-index sources configured"**: Check your environment variables are set correctly
+6. **"Auto-indexing configuration error"**: Verify the environment variable format uses `AUTO_INDEXING_SOURCES_N_URI`
 
 ### Checking Index Status
 

{kodit-0.2.9 → kodit-0.3.0}/src/kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.9'
-__version_tuple__ = version_tuple = (0, 2, 9)
+__version__ = version = '0.3.0'
+__version_tuple__ = version_tuple = (0, 3, 0)

kodit-0.3.0/src/kodit/app.py

@@ -0,0 +1,70 @@
+"""FastAPI application for kodit API."""
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from asgi_correlation_id import CorrelationIdMiddleware
+from fastapi import FastAPI
+
+from kodit.config import AppContext
+from kodit.infrastructure.indexing.auto_indexing_service import AutoIndexingService
+from kodit.mcp import mcp
+from kodit.middleware import ASGICancelledErrorMiddleware, logging_middleware
+
+# Global auto-indexing service
+_auto_indexing_service: AutoIndexingService | None = None
+
+
+@asynccontextmanager
+async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
+    """Manage application lifespan for auto-indexing."""
+    global _auto_indexing_service  # noqa: PLW0603
+    # Start auto-indexing service
+    app_context = AppContext()
+    db = await app_context.get_db()
+    _auto_indexing_service = AutoIndexingService(
+        app_context=app_context,
+        session_factory=db.session_factory,
+    )
+    await _auto_indexing_service.start_background_indexing()
+    yield
+    if _auto_indexing_service:
+        await _auto_indexing_service.stop()
+
+
+# See https://gofastmcp.com/deployment/asgi#fastapi-integration
+mcp_app = mcp.sse_app()
+
+
+@asynccontextmanager
+async def combined_lifespan(app: FastAPI) -> AsyncIterator[None]:
+    """Combine app and MCP lifespans."""
+    async with app_lifespan(app), mcp_app.router.lifespan_context(app):
+        yield
+
+
+app = FastAPI(title="kodit API", lifespan=combined_lifespan)
+
+# Add middleware
+app.middleware("http")(logging_middleware)
+app.add_middleware(CorrelationIdMiddleware)
+
+
+@app.get("/")
+async def root() -> dict[str, str]:
+    """Return a welcome message for the kodit API."""
+    return {"message": "Hello, World!"}
+
+
+@app.get("/healthz")
+async def healthz() -> dict[str, str]:
+    """Return a health check for the kodit API."""
+    return {"status": "ok"}
+
+
+# Add mcp routes last, otherwise previous routes aren't added
+app.mount("", mcp_app)
+
+# Wrap the entire app with ASGI middleware after all routes are added to suppress
+# CancelledError at the ASGI level
+app = ASGICancelledErrorMiddleware(app)

{kodit-0.2.9 → kodit-0.3.0}/src/kodit/cli.py

@@ -59,12 +59,17 @@ def cli(
 
 @cli.command()
 @click.argument("sources", nargs=-1)
+@click.option(
+    "--auto-index", is_flag=True, help="Index all configured auto-index sources"
+)
 @with_app_context
 @with_session
 async def index(
     session: AsyncSession,
     app_context: AppContext,
     sources: list[str],
+    *,  # Force keyword-only arguments
+    auto_index: bool,
 ) -> None:
     """List indexes, or index data sources."""
     log = structlog.get_logger(__name__)
@@ -78,6 +83,16 @@ async def index(
         source_service=source_service,
     )
 
+    if auto_index:
+        log.info("Auto-indexing configuration", config=app_context.auto_indexing)
+        auto_sources = app_context.auto_indexing.sources
+        if not auto_sources:
+            click.echo("No auto-index sources configured.")
+            return
+
+        click.echo(f"Auto-indexing {len(auto_sources)} configured sources...")
+        sources = [source.uri for source in auto_sources]
+
     if not sources:
         log_event("kodit.cli.index.list")
         # No source specified, list all indexes

{kodit-0.2.9 → kodit-0.3.0}/src/kodit/config.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal, TypeVar
 
 import click
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 if TYPE_CHECKING:
@@ -37,11 +37,45 @@ class Endpoint(BaseModel):
 
 
 class Search(BaseModel):
-    """Search provides configuration for a search engine."""
+    """Search configuration."""
 
     provider: Literal["sqlite", "vectorchord"] = Field(default="sqlite")
 
 
+class AutoIndexingSource(BaseModel):
+    """Configuration for a single auto-indexing source."""
+
+    uri: str = Field(description="URI of the source to index (git URL or local path)")
+
+
+class AutoIndexingConfig(BaseModel):
+    """Configuration for auto-indexing."""
+
+    sources: list[AutoIndexingSource] = Field(
+        default_factory=list, description="List of sources to auto-index"
+    )
+
+    @field_validator("sources", mode="before")
+    @classmethod
+    def parse_sources(cls, v: Any) -> list[AutoIndexingSource]:
+        """Parse sources from environment variables or other formats."""
+        if v is None:
+            return []
+        if isinstance(v, list):
+            return v
+        if isinstance(v, dict):
+            # Handle case where env vars are numbered keys like {'0': {'uri': '...'}}
+            sources = []
+            i = 0
+            while str(i) in v:
+                source_data = v[str(i)]
+                if isinstance(source_data, dict) and "uri" in source_data:
+                    sources.append(AutoIndexingSource(uri=source_data["uri"]))
+                i += 1
+            return sources
+        return v
+
+
 class AppContext(BaseSettings):
     """Global context for the kodit project. Provides a shared state for the app."""
 
@@ -50,7 +84,7 @@ class AppContext(BaseSettings):
         env_file_encoding="utf-8",
         env_nested_delimiter="_",
         nested_model_default_partial_update=True,
-        env_nested_max_split=1,
+        extra="ignore",
     )
 
     data_dir: Path = Field(default=DEFAULT_BASE_DIR)
@@ -76,6 +110,9 @@ class AppContext(BaseSettings):
     default_search: Search = Field(
         default=Search(),
     )
+    auto_indexing: AutoIndexingConfig | None = Field(
+        default=AutoIndexingConfig(), description="Auto-indexing configuration"
+    )
     _db: Database | None = None
 
     def model_post_init(self, _: Any) -> None:

kodit-0.3.0/src/kodit/infrastructure/indexing/auto_indexing_service.py

@@ -0,0 +1,84 @@
+"""Service for automatically indexing configured sources."""
+
+import asyncio
+from collections.abc import Callable
+from contextlib import suppress
+
+import structlog
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.application.factories.code_indexing_factory import (
+    create_code_indexing_application_service,
+)
+from kodit.config import AppContext
+from kodit.domain.services.source_service import SourceService
+
+
+class AutoIndexingService:
+    """Service for automatically indexing configured sources."""
+
+    def __init__(
+        self,
+        app_context: AppContext,
+        session_factory: Callable[[], AsyncSession],
+    ) -> None:
+        """Initialize the auto-indexing service."""
+        self.app_context = app_context
+        self.session_factory = session_factory
+        self.log = structlog.get_logger(__name__)
+        self._indexing_task: asyncio.Task | None = None
+
+    async def start_background_indexing(self) -> None:
+        """Start background indexing of configured sources."""
+        if (
+            not self.app_context.auto_indexing
+            or len(self.app_context.auto_indexing.sources) == 0
+        ):
+            self.log.info("Auto-indexing is disabled (no sources configured)")
+            return
+
+        auto_sources = [source.uri for source in self.app_context.auto_indexing.sources]
+        self.log.info("Starting background indexing", num_sources=len(auto_sources))
+        self._indexing_task = asyncio.create_task(self._index_sources(auto_sources))
+
+    async def _index_sources(self, sources: list[str]) -> None:
+        """Index all configured sources in the background."""
+        async with self.session_factory() as session:
+            source_service = SourceService(
+                clone_dir=self.app_context.get_clone_dir(),
+                session_factory=lambda: session,
+            )
+
+            service = create_code_indexing_application_service(
+                app_context=self.app_context,
+                session=session,
+                source_service=source_service,
+            )
+
+            for source in sources:
+                try:
+                    self.log.info("Auto-indexing source", source=source)
+
+                    # Create source
+                    s = await source_service.create(source)
+
+                    # Create index
+                    index = await service.create_index(s.id)
+
+                    # Run indexing (without progress callback for background mode)
+                    await service.run_index(index.id, progress_callback=None)
+
+                    self.log.info("Successfully auto-indexed source", source=source)
+
+                except Exception as exc:
+                    self.log.exception(
+                        "Failed to auto-index source", source=source, error=str(exc)
+                    )
+                    # Continue with other sources even if one fails
+
+    async def stop(self) -> None:
+        """Stop background indexing."""
+        if self._indexing_task:
+            self._indexing_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await self._indexing_task

kodit-0.3.0/tests/kodit/config_test.py

@@ -0,0 +1,73 @@
+"""Tests for configuration classes."""
+
+import os
+
+from kodit.config import AppContext, AutoIndexingConfig, AutoIndexingSource
+
+
+class TestAutoIndexingSource:
+    """Test the AutoIndexingSource configuration class."""
+
+    def test_auto_indexing_source_creation(self) -> None:
+        """Test creating an AutoIndexingSource."""
+        source = AutoIndexingSource(uri="https://github.com/test/repo")
+        assert source.uri == "https://github.com/test/repo"
+
+
+class TestAutoIndexingConfig:
+    """Test the AutoIndexingConfig configuration class."""
+
+    def test_auto_indexing_config_empty(self) -> None:
+        """Test empty auto-indexing configuration."""
+        config = AutoIndexingConfig()
+        assert config.sources == []
+
+    def test_auto_indexing_config_with_sources(self) -> None:
+        """Test auto-indexing configuration with sources."""
+        sources = [
+            AutoIndexingSource(uri="https://github.com/test/repo1"),
+            AutoIndexingSource(uri="https://github.com/test/repo2"),
+        ]
+        config = AutoIndexingConfig(sources=sources)
+        assert len(config.sources) == 2
+        assert config.sources[0].uri == "https://github.com/test/repo1"
+        assert config.sources[1].uri == "https://github.com/test/repo2"
+
+
+class TestAppContextAutoIndexing:
+    """Test auto-indexing functionality in AppContext."""
+
+    def test_get_auto_index_sources_empty(self) -> None:
+        """Test getting auto-index sources when none are configured."""
+        app_context = AppContext()
+        sources = app_context.auto_indexing.sources
+        assert sources == []
+
+    def test_get_auto_index_sources_with_config(self) -> None:
+        """Test getting auto-index sources when configured."""
+        auto_sources = [
+            AutoIndexingSource(uri="https://github.com/test/repo1"),
+            AutoIndexingSource(uri="/local/path/to/repo"),
+        ]
+        app_context = AppContext(auto_indexing=AutoIndexingConfig(sources=auto_sources))
+        sources = app_context.auto_indexing.sources
+        assert sources == auto_sources
+
+    def test_auto_indexing_from_environment_variables(self) -> None:
+        """Test auto-indexing configuration from environment variables."""
+        # Set environment variables for auto-indexing
+        os.environ["AUTO_INDEXING_SOURCES_0_URI"] = "https://github.com/test/repo1"
+        os.environ["AUTO_INDEXING_SOURCES_1_URI"] = "https://github.com/test/repo2"
+
+        try:
+            app_context = AppContext()
+            sources = app_context.auto_indexing.sources
+            uris = [source.uri for source in sources]
+            assert uris == [
+                "https://github.com/test/repo1",
+                "https://github.com/test/repo2",
+            ]
+        finally:
+            # Clean up environment variables
+            del os.environ["AUTO_INDEXING_SOURCES_0_URI"]
+            del os.environ["AUTO_INDEXING_SOURCES_1_URI"]