kodit 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

kodit/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.3'
-__version_tuple__ = version_tuple = (0, 3, 3)
+__version__ = version = '0.3.5'
+__version_tuple__ = version_tuple = (0, 3, 5)

kodit/app.py

@@ -6,28 +6,47 @@ from contextlib import asynccontextmanager
 from asgi_correlation_id import CorrelationIdMiddleware
 from fastapi import FastAPI
 
+from kodit.application.services.sync_scheduler import SyncSchedulerService
 from kodit.config import AppContext
 from kodit.infrastructure.indexing.auto_indexing_service import AutoIndexingService
 from kodit.mcp import mcp
 from kodit.middleware import ASGICancelledErrorMiddleware, logging_middleware
 
-# Global auto-indexing service
+# Global services
 _auto_indexing_service: AutoIndexingService | None = None
+_sync_scheduler_service: SyncSchedulerService | None = None
 
 
 @asynccontextmanager
 async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
-    """Manage application lifespan for auto-indexing."""
-    global _auto_indexing_service  # noqa: PLW0603
-    # Start auto-indexing service
+    """Manage application lifespan for auto-indexing and sync."""
+    global _auto_indexing_service, _sync_scheduler_service  # noqa: PLW0603
+
     app_context = AppContext()
     db = await app_context.get_db()
+
+    # Start auto-indexing service
     _auto_indexing_service = AutoIndexingService(
         app_context=app_context,
         session_factory=db.session_factory,
     )
     await _auto_indexing_service.start_background_indexing()
+
+    # Start sync scheduler service
+    if app_context.sync.enabled:
+        _sync_scheduler_service = SyncSchedulerService(
+            app_context=app_context,
+            session_factory=db.session_factory,
+        )
+        _sync_scheduler_service.start_periodic_sync(
+            interval_seconds=app_context.sync.interval_seconds
+        )
+
     yield
+
+    # Stop services
+    if _sync_scheduler_service:
+        await _sync_scheduler_service.stop_periodic_sync()
     if _auto_indexing_service:
         await _auto_indexing_service.stop()
 

kodit/application/factories/code_indexing_factory.py

@@ -13,7 +13,7 @@ from kodit.domain.services.index_query_service import IndexQueryService
 from kodit.domain.services.index_service import (
     IndexDomainService,
 )
-from kodit.domain.value_objects import LanguageMapping, SnippetExtractionStrategy
+from kodit.domain.value_objects import LanguageMapping
 from kodit.infrastructure.bm25.bm25_factory import bm25_repository_factory
 from kodit.infrastructure.embedding.embedding_factory import (
     embedding_domain_service_factory,
@@ -31,15 +31,9 @@ from kodit.infrastructure.enrichment.null_enrichment_provider import (
     NullEnrichmentProvider,
 )
 from kodit.infrastructure.indexing.fusion_service import ReciprocalRankFusionService
-from kodit.infrastructure.snippet_extraction.factories import (
-    create_snippet_query_provider,
-)
-from kodit.infrastructure.snippet_extraction.language_detection_service import (
+from kodit.infrastructure.slicing.language_detection_service import (
     FileSystemLanguageDetectionService,
 )
-from kodit.infrastructure.snippet_extraction.tree_sitter_snippet_extractor import (
-    TreeSitterSnippetExtractor,
-)
 from kodit.infrastructure.sqlalchemy.embedding_repository import (
     SqlAlchemyEmbeddingRepository,
 )
@@ -63,17 +57,9 @@ def create_code_indexing_application_service(
 
     # Create infrastructure services
     language_detector = FileSystemLanguageDetectionService(language_map)
-    query_provider = create_snippet_query_provider()
 
-    # Create snippet extractors
-    method_extractor = TreeSitterSnippetExtractor(query_provider)
-
-    snippet_extractors = {
-        SnippetExtractionStrategy.METHOD_BASED: method_extractor,
-    }
     index_domain_service = IndexDomainService(
         language_detector=language_detector,
-        snippet_extractors=snippet_extractors,
         enrichment_service=enrichment_service,
         clone_dir=app_context.get_clone_dir(),
     )
@@ -136,17 +122,9 @@ def create_fast_test_code_indexing_application_service(
 
     # Create infrastructure services
     language_detector = FileSystemLanguageDetectionService(language_map)
-    query_provider = create_snippet_query_provider()
-
-    # Create snippet extractors
-    method_extractor = TreeSitterSnippetExtractor(query_provider)
 
-    snippet_extractors = {
-        SnippetExtractionStrategy.METHOD_BASED: method_extractor,
-    }
     index_domain_service = IndexDomainService(
         language_detector=language_detector,
-        snippet_extractors=snippet_extractors,
         enrichment_service=enrichment_service,
         clone_dir=app_context.get_clone_dir(),
     )

kodit/application/services/code_indexing_application_service.py

@@ -100,6 +100,11 @@ class CodeIndexingApplicationService:
             self.log.info("No new changes to index", index_id=index.id)
             return
 
+        # Delete the old snippets from the files that have changed
+        await self.index_repository.delete_snippets_by_file_ids(
+            [file.id for file in index.source.working_copy.changed_files() if file.id]
+        )
+
         # Extract and create snippets (domain service handles progress)
         self.log.info("Creating snippets for files", index_id=index.id)
         index = await self.index_domain_service.extract_snippets_from_index(
@@ -115,6 +120,9 @@ class CodeIndexingApplicationService:
             msg = f"Index {index.id} not found after snippet extraction"
             raise ValueError(msg)
         index = flushed_index
+        if len(index.snippets) == 0:
+            self.log.info("No snippets to index after extraction", index_id=index.id)
+            return
 
         # Create BM25 index
         self.log.info("Creating keyword index")
@@ -154,8 +162,8 @@ class CodeIndexingApplicationService:
         # Apply filters if provided
         filtered_snippet_ids: list[int] | None = None
         if request.filters:
-            # Use domain service for filtering
-            prefilter_request = replace(request, top_k=None)
+            # Use domain service for filtering (use large top_k for pre-filtering)
+            prefilter_request = replace(request, top_k=10000)
             snippet_results = await self.index_query_service.search_snippets(
                 prefilter_request
             )

kodit/application/services/sync_scheduler.py

@@ -0,0 +1,128 @@
+"""Service for scheduling periodic sync operations."""
+
+import asyncio
+from collections.abc import Callable
+from contextlib import suppress
+
+import structlog
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.application.factories.code_indexing_factory import (
+    create_code_indexing_application_service,
+)
+from kodit.config import AppContext
+from kodit.domain.services.index_query_service import IndexQueryService
+from kodit.infrastructure.indexing.fusion_service import ReciprocalRankFusionService
+from kodit.infrastructure.sqlalchemy.index_repository import SqlAlchemyIndexRepository
+
+
+class SyncSchedulerService:
+    """Service for scheduling periodic sync operations."""
+
+    def __init__(
+        self,
+        app_context: AppContext,
+        session_factory: Callable[[], AsyncSession],
+    ) -> None:
+        """Initialize the sync scheduler service."""
+        self.app_context = app_context
+        self.session_factory = session_factory
+        self.log = structlog.get_logger(__name__)
+        self._sync_task: asyncio.Task | None = None
+        self._shutdown_event = asyncio.Event()
+
+    def start_periodic_sync(self, interval_seconds: float = 1800) -> None:
+        """Start periodic sync of all indexes."""
+        self.log.info("Starting periodic sync", interval_seconds=interval_seconds)
+
+        self._sync_task = asyncio.create_task(self._sync_loop(interval_seconds))
+
+    async def stop_periodic_sync(self) -> None:
+        """Stop the periodic sync task."""
+        self.log.info("Stopping periodic sync")
+        self._shutdown_event.set()
+
+        if self._sync_task and not self._sync_task.done():
+            self._sync_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await self._sync_task
+
+    async def _sync_loop(self, interval_seconds: float) -> None:
+        """Run the sync loop at the specified interval."""
+        while not self._shutdown_event.is_set():
+            try:
+                await self._perform_sync()
+            except Exception as e:
+                self.log.exception("Sync operation failed", error=e)
+
+            # Wait for the interval or until shutdown
+            try:
+                await asyncio.wait_for(
+                    self._shutdown_event.wait(), timeout=interval_seconds
+                )
+                # If we reach here, shutdown was requested
+                break
+            except TimeoutError:
+                # Continue to next sync cycle
+                continue
+
+    async def _perform_sync(self) -> None:
+        """Perform a sync operation on all indexes."""
+        self.log.info("Starting sync operation")
+
+        async with self.session_factory() as session:
+            # Create services
+            service = create_code_indexing_application_service(
+                app_context=self.app_context,
+                session=session,
+            )
+            index_query_service = IndexQueryService(
+                index_repository=SqlAlchemyIndexRepository(session=session),
+                fusion_service=ReciprocalRankFusionService(),
+            )
+
+            # Get all existing indexes
+            all_indexes = await index_query_service.list_indexes()
+
+            if not all_indexes:
+                self.log.info("No indexes found to sync")
+                return
+
+            self.log.info("Syncing indexes", count=len(all_indexes))
+
+            success_count = 0
+            failure_count = 0
+
+            # Sync each index
+            for index in all_indexes:
+                try:
+                    self.log.info(
+                        "Syncing index",
+                        index_id=index.id,
+                        source=str(index.source.working_copy.remote_uri),
+                    )
+
+                    await service.run_index(index, progress_callback=None)
+                    success_count += 1
+
+                    self.log.info(
+                        "Index sync completed",
+                        index_id=index.id,
+                        source=str(index.source.working_copy.remote_uri),
+                    )
+
+                except Exception as e:
+                    failure_count += 1
+                    self.log.exception(
+                        "Index sync failed",
+                        index_id=index.id,
+                        source=str(index.source.working_copy.remote_uri),
+                        error=e,
+                    )
+
+            self.log.info(
+                "Sync operation completed",
+                total=len(all_indexes),
+                success=success_count,
+                failures=failure_count,
+            )

kodit/cli.py

@@ -63,11 +63,105 @@ def cli(
     ctx.obj = config
 
 
+async def _handle_auto_index(
+    app_context: AppContext,
+    sources: list[str],  # noqa: ARG001
+) -> list[str]:
+    """Handle auto-index option and return sources to process."""
+    log = structlog.get_logger(__name__)
+    log.info("Auto-indexing configuration", config=app_context.auto_indexing)
+    if not app_context.auto_indexing or not app_context.auto_indexing.sources:
+        click.echo("No auto-index sources configured.")
+        return []
+    auto_sources = app_context.auto_indexing.sources
+    click.echo(f"Auto-indexing {len(auto_sources)} configured sources...")
+    return [source.uri for source in auto_sources]
+
+
+async def _handle_sync(
+    service: Any,
+    index_query_service: IndexQueryService,
+    sources: list[str],
+) -> None:
+    """Handle sync operation."""
+    log = structlog.get_logger(__name__)
+    log_event("kodit.cli.index.sync")
+
+    # Get all existing indexes
+    all_indexes = await index_query_service.list_indexes()
+
+    if not all_indexes:
+        click.echo("No existing indexes found to sync.")
+        return
+
+    # Filter indexes if specific sources are provided
+    indexes_to_sync = all_indexes
+    if sources:
+        # Filter indexes that match the provided sources
+        source_uris = set(sources)
+        indexes_to_sync = [
+            index for index in all_indexes
+            if str(index.source.working_copy.remote_uri) in source_uris
+        ]
+
+        if not indexes_to_sync:
+            click.echo(
+                f"No indexes found for the specified sources: {', '.join(sources)}"
+            )
+            return
+
+    click.echo(f"Syncing {len(indexes_to_sync)} indexes...")
+
+    # Sync each index
+    for index in indexes_to_sync:
+        click.echo(f"Syncing: {index.source.working_copy.remote_uri}")
+
+        # Create progress callback for this sync operation
+        progress_callback = create_multi_stage_progress_callback()
+
+        try:
+            await service.run_index(index, progress_callback)
+            click.echo(f"✓ Sync completed: {index.source.working_copy.remote_uri}")
+        except Exception as e:
+            log.exception("Sync failed", index_id=index.id, error=e)
+            click.echo(
+                f"✗ Sync failed: {index.source.working_copy.remote_uri} - {e}"
+            )
+
+
+async def _handle_list_indexes(index_query_service: IndexQueryService) -> None:
+    """Handle listing all indexes."""
+    log_event("kodit.cli.index.list")
+    # No source specified, list all indexes
+    indexes = await index_query_service.list_indexes()
+    headers: list[str | Cell] = [
+        "ID",
+        "Created At",
+        "Updated At",
+        "Source",
+        "Num Snippets",
+    ]
+    data = [
+        [
+            index.id,
+            index.created_at,
+            index.updated_at,
+            index.source.working_copy.remote_uri,
+            len(index.source.working_copy.files),
+        ]
+        for index in indexes
+    ]
+    click.echo(Table(headers=headers, data=data))
+
+
 @cli.command()
 @click.argument("sources", nargs=-1)
 @click.option(
     "--auto-index", is_flag=True, help="Index all configured auto-index sources"
 )
+@click.option(
+    "--sync", is_flag=True, help="Sync existing indexes with their remotes"
+)
 @with_app_context
 @with_session
 async def index(
@@ -76,8 +170,9 @@ async def index(
     sources: list[str],
     *,  # Force keyword-only arguments
     auto_index: bool,
+    sync: bool,
 ) -> None:
-    """List indexes, or index data sources."""
+    """List indexes, index data sources, or sync existing indexes."""
     log = structlog.get_logger(__name__)
     service = create_code_indexing_application_service(
         app_context=app_context,
@@ -89,36 +184,16 @@ async def index(
     )
 
     if auto_index:
-        log.info("Auto-indexing configuration", config=app_context.auto_indexing)
-        if not app_context.auto_indexing or not app_context.auto_indexing.sources:
-            click.echo("No auto-index sources configured.")
+        sources = await _handle_auto_index(app_context, sources)
+        if not sources:
             return
-        auto_sources = app_context.auto_indexing.sources
-        click.echo(f"Auto-indexing {len(auto_sources)} configured sources...")
-        sources = [source.uri for source in auto_sources]
+
+    if sync:
+        await _handle_sync(service, index_query_service, sources)
+        return
 
     if not sources:
-        log_event("kodit.cli.index.list")
-        # No source specified, list all indexes
-        indexes = await index_query_service.list_indexes()
-        headers: list[str | Cell] = [
-            "ID",
-            "Created At",
-            "Updated At",
-            "Source",
-            "Num Snippets",
-        ]
-        data = [
-            [
-                index.id,
-                index.created_at,
-                index.updated_at,
-                index.source.working_copy.remote_uri,
-                len(index.source.working_copy.files),
-            ]
-            for index in indexes
-        ]
-        click.echo(Table(headers=headers, data=data))
+        await _handle_list_indexes(index_query_service)
         return
     # Handle source indexing
     for source in sources:

kodit/config.py

@@ -81,6 +81,18 @@ class AutoIndexingConfig(BaseModel):
         return v
 
 
+class PeriodicSyncConfig(BaseModel):
+    """Configuration for periodic/scheduled syncing."""
+
+    enabled: bool = Field(default=True, description="Enable periodic sync")
+    interval_seconds: float = Field(
+        default=1800, description="Interval between automatic syncs in seconds"
+    )
+    retry_attempts: int = Field(
+        default=3, description="Number of retry attempts for failed syncs"
+    )
+
+
 class CustomAutoIndexingEnvSource(EnvSettingsSource):
     """Custom environment source for parsing AutoIndexingConfig."""
 
@@ -173,6 +185,9 @@ class AppContext(BaseSettings):
     auto_indexing: AutoIndexingConfig | None = Field(
         default=AutoIndexingConfig(), description="Auto-indexing configuration"
     )
+    periodic_sync: PeriodicSyncConfig = Field(
+        default=PeriodicSyncConfig(), description="Periodic sync configuration"
+    )
     _db: Database | None = None
 
     def model_post_init(self, _: Any) -> None:

kodit/domain/services/index_service.py

@@ -1,7 +1,6 @@
 """Pure domain service for Index aggregate operations."""
 
 from abc import ABC, abstractmethod
-from collections.abc import Mapping
 from pathlib import Path
 
 import structlog
@@ -13,14 +12,13 @@ from kodit.domain.services.enrichment_service import EnrichmentDomainService
 from kodit.domain.value_objects import (
     EnrichmentIndexRequest,
     EnrichmentRequest,
-    SnippetExtractionRequest,
-    SnippetExtractionResult,
-    SnippetExtractionStrategy,
+    LanguageMapping,
 )
 from kodit.infrastructure.cloning.git.working_copy import GitWorkingCopyProvider
 from kodit.infrastructure.cloning.metadata import FileMetadataExtractor
 from kodit.infrastructure.git.git_utils import is_valid_clone_target
 from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
+from kodit.infrastructure.slicing.slicer import Slicer
 from kodit.reporting import Reporter
 from kodit.utils.path_utils import path_from_uri
 
@@ -33,14 +31,6 @@ class LanguageDetectionService(ABC):
         """Detect the programming language of a file."""
 
 
-class SnippetExtractor(ABC):
-    """Abstract interface for snippet extraction."""
-
-    @abstractmethod
-    async def extract(self, file_path: Path, language: str) -> list[str]:
-        """Extract snippets from a file."""
-
-
 class IndexDomainService:
     """Pure domain service for Index aggregate operations.
 
@@ -54,14 +44,12 @@ class IndexDomainService:
     def __init__(
         self,
         language_detector: LanguageDetectionService,
-        snippet_extractors: Mapping[SnippetExtractionStrategy, SnippetExtractor],
         enrichment_service: EnrichmentDomainService,
         clone_dir: Path,
     ) -> None:
         """Initialize the index domain service."""
         self._clone_dir = clone_dir
         self._language_detector = language_detector
-        self._snippet_extractors = snippet_extractors
         self._enrichment_service = enrichment_service
         self.log = structlog.get_logger(__name__)
 
@@ -99,7 +87,6 @@ class IndexDomainService:
     async def extract_snippets_from_index(
         self,
         index: domain_entities.Index,
-        strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED,
         progress_callback: ProgressCallback | None = None,
     ) -> domain_entities.Index:
         """Extract code snippets from files in the index."""
@@ -109,46 +96,40 @@ class IndexDomainService:
             "Extracting snippets",
             index_id=index.id,
             file_count=file_count,
-            strategy=strategy.value,
         )
 
         # Only create snippets for files that have been added or modified
         files = index.source.working_copy.changed_files()
         index.delete_snippets_for_files(files)
 
-        reporter = Reporter(self.log, progress_callback)
-        await reporter.start(
-            "extract_snippets", len(files), "Extracting code snippets..."
-        )
-
-        new_snippets = []
-        for i, domain_file in enumerate(files, 1):
+        # Create a set of languages to extract snippets for
+        extensions = {file.extension() for file in files}
+        languages = []
+        for ext in extensions:
             try:
-                # Extract snippets from file
-                request = SnippetExtractionRequest(
-                    file_path=domain_file.as_path(), strategy=strategy
-                )
-                result = await self._extract_snippets(request)
-                for snippet_text in result.snippets:
-                    snippet = domain_entities.Snippet(
-                        derives_from=[domain_file],
-                    )
-                    snippet.add_original_content(snippet_text, result.language)
-                    new_snippets.append(snippet)
-
-            except (OSError, ValueError) as e:
-                self.log.debug(
-                    "Skipping file for snippet extraction",
-                    file_uri=str(domain_file.uri),
-                    error=str(e),
-                )
+                languages.append(LanguageMapping.get_language_for_extension(ext))
+            except ValueError as e:
+                self.log.info("Skipping", error=str(e))
                 continue
 
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "extract_snippets",
+            len(files) * len(languages),
+            "Extracting code snippets...",
+        )
+        # Calculate snippets for each language
+        slicer = Slicer()
+        for i, language in enumerate(languages):
             await reporter.step(
-                "extract_snippets", i, len(files), f"Processed {domain_file.uri.path}"
+                "extract_snippets",
+                len(files) * (i + 1),
+                len(files) * len(languages),
+                "Extracting code snippets...",
             )
+            s = slicer.extract_snippets(files, language=language)
+            index.snippets.extend(s)
 
-        index.snippets.extend(new_snippets)
         await reporter.done("extract_snippets")
         return index
 
@@ -187,28 +168,6 @@ class IndexDomainService:
         await reporter.done("enrichment")
         return list(snippet_map.values())
 
-    async def _extract_snippets(
-        self, request: SnippetExtractionRequest
-    ) -> SnippetExtractionResult:
-        # Domain logic: validate file exists
-        if not request.file_path.exists():
-            raise ValueError(f"File does not exist: {request.file_path}")
-
-        # Domain logic: detect language
-        language = await self._language_detector.detect_language(request.file_path)
-
-        # Domain logic: choose strategy and extractor
-        if request.strategy not in self._snippet_extractors:
-            raise ValueError(f"Unsupported extraction strategy: {request.strategy}")
-
-        extractor = self._snippet_extractors[request.strategy]
-        snippets = await extractor.extract(request.file_path, language)
-
-        # Domain logic: filter out empty snippets
-        filtered_snippets = [snippet for snippet in snippets if snippet.strip()]
-
-        return SnippetExtractionResult(snippets=filtered_snippets, language=language)
-
     def sanitize_uri(
         self, uri_or_path_like: str
     ) -> tuple[AnyUrl, domain_entities.SourceType]:
@@ -297,7 +256,7 @@ class IndexDomainService:
                     await metadata_extractor.extract(file_path=file_path)
                 )
             except (OSError, ValueError) as e:
-                self.log.info("Skipping file", file=str(file_path), error=str(e))
+                self.log.debug("Skipping file", file=str(file_path), error=str(e))
                 continue
 
         # Finally check if there are any modified files

kodit/domain/value_objects.py

@@ -134,14 +134,6 @@ class SearchType(Enum):
     HYBRID = "hybrid"
 
 
-@dataclass
-class SnippetExtractionResult:
-    """Domain model for snippet extraction result."""
-
-    snippets: list[str]
-    language: str
-
-
 @dataclass
 class Document:
     """Generic document model for indexing."""
@@ -640,20 +632,6 @@ class SnippetQuery(BaseModel):
     top_k: int = 10
 
 
-class SnippetExtractionStrategy(str, Enum):
-    """Different strategies for extracting snippets from files."""
-
-    METHOD_BASED = "method_based"
-
-
-@dataclass
-class SnippetExtractionRequest:
-    """Domain model for snippet extraction request."""
-
-    file_path: Path
-    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED
-
-
 class FileProcessingStatus(IntEnum):
     """File processing status."""
 
@@ -661,3 +639,13 @@ class FileProcessingStatus(IntEnum):
     ADDED = 1
     MODIFIED = 2
     DELETED = 3
+
+
+@dataclass
+class FunctionDefinition:
+    """Cached function definition."""
+
+    name: str
+    qualified_name: str
+    start_byte: int
+    end_byte: int