kodit 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

kodit/cli.py

@@ -1,5 +1,6 @@
 """Command line interface for kodit."""
 
+import asyncio
 import signal
 from pathlib import Path
 from typing import Any
@@ -10,19 +11,56 @@ import uvicorn
 from pytable_formatter import Cell, Table
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from kodit.bm25.keyword_search_factory import keyword_search_factory
+from kodit.application.services.snippet_application_service import (
+    SnippetApplicationService,
+)
 from kodit.config import (
     AppContext,
     with_app_context,
     with_session,
 )
-from kodit.embedding.embedding_factory import embedding_factory
-from kodit.enrichment.enrichment_factory import enrichment_factory
-from kodit.indexing.indexing_repository import IndexRepository
-from kodit.indexing.indexing_service import IndexService, SearchRequest
+from kodit.domain.errors import EmptySourceError
+from kodit.domain.services.source_service import SourceService
+from kodit.domain.value_objects import MultiSearchRequest
+from kodit.infrastructure.indexing.indexing_factory import (
+    create_indexing_application_service,
+)
+from kodit.infrastructure.snippet_extraction.snippet_extraction_factory import (
+    create_snippet_extraction_domain_service,
+    create_snippet_repositories,
+)
+from kodit.infrastructure.ui.progress import (
+    create_lazy_progress_callback,
+    create_multi_stage_progress_callback,
+)
 from kodit.log import configure_logging, configure_telemetry, log_event
-from kodit.source.source_repository import SourceRepository
-from kodit.source.source_service import SourceService
+
+
+def create_snippet_application_service(
+    session: AsyncSession,
+) -> SnippetApplicationService:
+    """Create a snippet application service with all dependencies.
+
+    Args:
+        session: SQLAlchemy session
+
+    Returns:
+        Configured snippet application service
+
+    """
+    # Create domain service
+    snippet_extraction_service = create_snippet_extraction_domain_service()
+
+    # Create repositories
+    snippet_repository, file_repository = create_snippet_repositories(session)
+
+    # Create application service
+    return SnippetApplicationService(
+        snippet_extraction_service=snippet_extraction_service,
+        snippet_repository=snippet_repository,
+        file_repository=file_repository,
+        session=session,
+    )
 
 
 @click.group(context_settings={"max_content_width": 100})
@@ -64,20 +102,17 @@ async def index(
     sources: list[str],
 ) -> None:
     """List indexes, or index data sources."""
-    source_repository = SourceRepository(session)
-    source_service = SourceService(app_context.get_clone_dir(), source_repository)
-    repository = IndexRepository(session)
-    service = IndexService(
-        repository=repository,
+    log = structlog.get_logger(__name__)
+    source_service = SourceService(
+        clone_dir=app_context.get_clone_dir(),
+        session_factory=lambda: session,
+    )
+    snippet_service = create_snippet_application_service(session)
+    service = create_indexing_application_service(
+        app_context=app_context,
+        session=session,
         source_service=source_service,
-        keyword_search_provider=keyword_search_factory(app_context, session),
-        code_search_service=embedding_factory(
-            task_name="code", app_context=app_context, session=session
-        ),
-        text_search_service=embedding_factory(
-            task_name="text", app_context=app_context, session=session
-        ),
-        enrichment_service=enrichment_factory(app_context),
+        snippet_application_service=snippet_service,
     )
 
     if not sources:
@@ -109,11 +144,28 @@ async def index(
             msg = "File indexing is not implemented yet"
             raise click.UsageError(msg)
 
-        # Index source
+        # Index source with progress
         log_event("kodit.cli.index.create")
-        s = await source_service.create(source)
-        index = await service.create(s.id)
-        await service.run(index.id)
+
+        # Create a lazy progress callback that only shows progress when needed
+        progress_callback = create_lazy_progress_callback()
+        s = await source_service.create(source, progress_callback)
+
+        index = await service.create_index(s.id)
+
+        # Create a new progress callback for the indexing operations
+        indexing_progress_callback = create_multi_stage_progress_callback()
+        try:
+            await service.run_index(index.id, indexing_progress_callback)
+        except EmptySourceError as e:
+            log.exception("Empty source error", error=e)
+            msg = f"""{e}. This could mean:
+• The repository contains no supported file types
+• All files are excluded by ignore patterns
+• The files contain no extractable code snippets
+Please check the repository contents and try again.
+"""
+            click.echo(msg)
 
 
 @cli.group()
@@ -137,23 +189,19 @@ async def code(
     This works best if your query is code.
     """
     log_event("kodit.cli.search.code")
-    source_repository = SourceRepository(session)
-    source_service = SourceService(app_context.get_clone_dir(), source_repository)
-    repository = IndexRepository(session)
-    service = IndexService(
-        repository=repository,
+    source_service = SourceService(
+        clone_dir=app_context.get_clone_dir(),
+        session_factory=lambda: session,
+    )
+    snippet_service = create_snippet_application_service(session)
+    service = create_indexing_application_service(
+        app_context=app_context,
+        session=session,
         source_service=source_service,
-        keyword_search_provider=keyword_search_factory(app_context, session),
-        code_search_service=embedding_factory(
-            task_name="code", app_context=app_context, session=session
-        ),
-        text_search_service=embedding_factory(
-            task_name="text", app_context=app_context, session=session
-        ),
-        enrichment_service=enrichment_factory(app_context),
+        snippet_application_service=snippet_service,
     )
 
-    snippets = await service.search(SearchRequest(code_query=query, top_k=top_k))
+    snippets = await service.search(MultiSearchRequest(code_query=query, top_k=top_k))
 
     if len(snippets) == 0:
         click.echo("No snippets found")
@@ -181,23 +229,19 @@ async def keyword(
 ) -> None:
     """Search for snippets using keyword search."""
     log_event("kodit.cli.search.keyword")
-    source_repository = SourceRepository(session)
-    source_service = SourceService(app_context.get_clone_dir(), source_repository)
-    repository = IndexRepository(session)
-    service = IndexService(
-        repository=repository,
+    source_service = SourceService(
+        clone_dir=app_context.get_clone_dir(),
+        session_factory=lambda: session,
+    )
+    snippet_service = create_snippet_application_service(session)
+    service = create_indexing_application_service(
+        app_context=app_context,
+        session=session,
         source_service=source_service,
-        keyword_search_provider=keyword_search_factory(app_context, session),
-        code_search_service=embedding_factory(
-            task_name="code", app_context=app_context, session=session
-        ),
-        text_search_service=embedding_factory(
-            task_name="text", app_context=app_context, session=session
-        ),
-        enrichment_service=enrichment_factory(app_context),
+        snippet_application_service=snippet_service,
     )
 
-    snippets = await service.search(SearchRequest(keywords=keywords, top_k=top_k))
+    snippets = await service.search(MultiSearchRequest(keywords=keywords, top_k=top_k))
 
     if len(snippets) == 0:
         click.echo("No snippets found")
@@ -228,23 +272,19 @@ async def text(
     This works best if your query is text.
     """
     log_event("kodit.cli.search.text")
-    source_repository = SourceRepository(session)
-    source_service = SourceService(app_context.get_clone_dir(), source_repository)
-    repository = IndexRepository(session)
-    service = IndexService(
-        repository=repository,
+    source_service = SourceService(
+        clone_dir=app_context.get_clone_dir(),
+        session_factory=lambda: session,
+    )
+    snippet_service = create_snippet_application_service(session)
+    service = create_indexing_application_service(
+        app_context=app_context,
+        session=session,
         source_service=source_service,
-        keyword_search_provider=keyword_search_factory(app_context, session),
-        code_search_service=embedding_factory(
-            task_name="code", app_context=app_context, session=session
-        ),
-        text_search_service=embedding_factory(
-            task_name="text", app_context=app_context, session=session
-        ),
-        enrichment_service=enrichment_factory(app_context),
+        snippet_application_service=snippet_service,
     )
 
-    snippets = await service.search(SearchRequest(text_query=query, top_k=top_k))
+    snippets = await service.search(MultiSearchRequest(text_query=query, top_k=top_k))
 
     if len(snippets) == 0:
         click.echo("No snippets found")
@@ -276,30 +316,26 @@ async def hybrid(  # noqa: PLR0913
 ) -> None:
     """Search for snippets using hybrid search."""
     log_event("kodit.cli.search.hybrid")
-    source_repository = SourceRepository(session)
-    source_service = SourceService(app_context.get_clone_dir(), source_repository)
-    repository = IndexRepository(session)
-    service = IndexService(
-        repository=repository,
+    source_service = SourceService(
+        clone_dir=app_context.get_clone_dir(),
+        session_factory=lambda: session,
+    )
+    snippet_service = create_snippet_application_service(session)
+    service = create_indexing_application_service(
+        app_context=app_context,
+        session=session,
         source_service=source_service,
-        keyword_search_provider=keyword_search_factory(app_context, session),
-        code_search_service=embedding_factory(
-            task_name="code", app_context=app_context, session=session
-        ),
-        text_search_service=embedding_factory(
-            task_name="text", app_context=app_context, session=session
-        ),
-        enrichment_service=enrichment_factory(app_context),
+        snippet_application_service=snippet_service,
     )
 
     # Parse keywords into a list of strings
     keywords_list = [k.strip().lower() for k in keywords.split(",")]
 
     snippets = await service.search(
-        SearchRequest(
-            text_query=text,
+        MultiSearchRequest(
             keywords=keywords_list,
             code_query=code,
+            text_query=text,
             top_k=top_k,
         )
     )
@@ -362,4 +398,4 @@ def version() -> None:
 
 
 if __name__ == "__main__":
-    cli()
+    asyncio.run(cli())

kodit/database.py

@@ -1,41 +1,19 @@
 """Database configuration for kodit."""
 
-from datetime import UTC, datetime
 from pathlib import Path
 
 import structlog
 from alembic import command
 from alembic.config import Config as AlembicConfig
-from sqlalchemy import DateTime
 from sqlalchemy.ext.asyncio import (
-    AsyncAttrs,
     AsyncSession,
     async_sessionmaker,
     create_async_engine,
 )
-from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 
 from kodit import migrations
 
 
-class Base(AsyncAttrs, DeclarativeBase):
-    """Base class for all models."""
-
-
-class CommonMixin:
-    """Common mixin for all models."""
-
-    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), default=lambda: datetime.now(UTC)
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True),
-        default=lambda: datetime.now(UTC),
-        onupdate=lambda: datetime.now(UTC),
-    )
-
-
 class Database:
     """Database class for kodit."""
 

kodit/domain/__init__.py

@@ -0,0 +1 @@
+"""Domain layer containing models, services, and repositories."""

kodit/{source/source_models.py → domain/entities.py}

@@ -1,24 +1,42 @@
-"""Source models for managing code sources.
+"""SQLAlchemy entities."""
 
-This module defines the SQLAlchemy models used for storing and managing code sources.
-It includes models for tracking different types of sources (git repositories and local
-folders) and their relationships.
-"""
-
-import datetime
-from enum import Enum as EnumType
+from datetime import UTC, datetime
+from enum import Enum
 
 from git import Actor
-from sqlalchemy import Enum, ForeignKey, Integer, String, UniqueConstraint
-from sqlalchemy.orm import Mapped, mapped_column
-
-from kodit.database import Base, CommonMixin
-
-# Enable proper type hints for SQLAlchemy models
-__all__ = ["File", "Source"]
+from sqlalchemy import (
+    DateTime,
+    ForeignKey,
+    Integer,
+    String,
+    UnicodeText,
+    UniqueConstraint,
+)
+from sqlalchemy import Enum as SQLAlchemyEnum
+from sqlalchemy.ext.asyncio import AsyncAttrs
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+from sqlalchemy.types import JSON
+
+
+class Base(AsyncAttrs, DeclarativeBase):
+    """Base class for all models."""
+
+
+class CommonMixin:
+    """Common mixin for all models."""
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), default=lambda: datetime.now(UTC)
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(UTC),
+        onupdate=lambda: datetime.now(UTC),
+    )
 
 
-class SourceType(EnumType):
+class SourceType(Enum):
     """The type of source."""
 
     UNKNOWN = 0
@@ -45,7 +63,7 @@ class Source(Base, CommonMixin):
     uri: Mapped[str] = mapped_column(String(1024), index=True, unique=True)
     cloned_path: Mapped[str] = mapped_column(String(1024), index=True)
     type: Mapped[SourceType] = mapped_column(
-        Enum(SourceType), default=SourceType.UNKNOWN, index=True
+        SQLAlchemyEnum(SourceType), default=SourceType.UNKNOWN, index=True
     )
 
     def __init__(self, uri: str, cloned_path: str, source_type: SourceType) -> None:
@@ -100,8 +118,8 @@ class File(Base, CommonMixin):
 
     def __init__(  # noqa: PLR0913
         self,
-        created_at: datetime.datetime,
-        updated_at: datetime.datetime,
+        created_at: datetime,
+        updated_at: datetime,
         source_id: int,
         cloned_path: str,
         mime_type: str = "",
@@ -119,3 +137,54 @@ class File(Base, CommonMixin):
         self.uri = uri
         self.sha256 = sha256
         self.size_bytes = size_bytes
+
+
+class EmbeddingType(Enum):
+    """Embedding type."""
+
+    CODE = 1
+    TEXT = 2
+
+
+class Embedding(Base, CommonMixin):
+    """Embedding model."""
+
+    __tablename__ = "embeddings"
+
+    snippet_id: Mapped[int] = mapped_column(ForeignKey("snippets.id"), index=True)
+    type: Mapped[EmbeddingType] = mapped_column(
+        SQLAlchemyEnum(EmbeddingType), index=True
+    )
+    embedding: Mapped[list[float]] = mapped_column(JSON)
+
+
+class Index(Base, CommonMixin):
+    """Index model."""
+
+    __tablename__ = "indexes"
+
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("sources.id"), unique=True, index=True
+    )
+
+    def __init__(self, source_id: int) -> None:
+        """Initialize the index."""
+        super().__init__()
+        self.source_id = source_id
+
+
+class Snippet(Base, CommonMixin):
+    """Snippet model."""
+
+    __tablename__ = "snippets"
+
+    file_id: Mapped[int] = mapped_column(ForeignKey("files.id"), index=True)
+    index_id: Mapped[int] = mapped_column(ForeignKey("indexes.id"), index=True)
+    content: Mapped[str] = mapped_column(UnicodeText, default="")
+
+    def __init__(self, file_id: int, index_id: int, content: str) -> None:
+        """Initialize the snippet."""
+        super().__init__()
+        self.file_id = file_id
+        self.index_id = index_id
+        self.content = content

kodit/domain/enums.py

@@ -0,0 +1,9 @@
+"""Domain enums."""
+
+from enum import Enum
+
+
+class SnippetExtractionStrategy(str, Enum):
+    """Different strategies for extracting snippets from files."""
+
+    METHOD_BASED = "method_based"

kodit/domain/errors.py

@@ -0,0 +1,5 @@
+"""Domain errors."""
+
+
+class EmptySourceError(ValueError):
+    """Error raised when a source is empty."""

kodit/domain/interfaces.py

@@ -0,0 +1,27 @@
+"""Domain interfaces."""
+
+from abc import ABC, abstractmethod
+
+from kodit.domain.value_objects import ProgressEvent
+
+
+class ProgressCallback(ABC):
+    """Abstract interface for progress callbacks."""
+
+    @abstractmethod
+    async def on_progress(self, event: ProgressEvent) -> None:
+        """On progress hook."""
+
+    @abstractmethod
+    async def on_complete(self, operation: str) -> None:
+        """On complete hook."""
+
+
+class NullProgressCallback(ProgressCallback):
+    """Null implementation of progress callback that does nothing."""
+
+    async def on_progress(self, event: ProgressEvent) -> None:
+        """Do nothing on progress."""
+
+    async def on_complete(self, operation: str) -> None:
+        """Do nothing on complete."""

kodit/domain/repositories.py

@@ -0,0 +1,95 @@
+"""Domain repositories with generic patterns."""
+
+from collections.abc import Sequence
+from typing import Protocol, TypeVar
+
+from kodit.domain.entities import (
+    Author,
+    AuthorFileMapping,
+    File,
+    Snippet,
+    Source,
+    SourceType,
+)
+
+T = TypeVar("T")
+
+
+class GenericRepository(Protocol[T]):
+    """Generic repository interface."""
+
+    async def get(self, id: int) -> T | None:  # noqa: A002
+        """Get entity by ID."""
+        ...
+
+    async def save(self, entity: T) -> T:
+        """Save entity."""
+        ...
+
+    async def delete(self, id: int) -> None:  # noqa: A002
+        """Delete entity by ID."""
+        ...
+
+    async def list(self) -> Sequence[T]:
+        """List all entities."""
+        ...
+
+
+class SourceRepository(GenericRepository[Source]):
+    """Source repository with specific methods."""
+
+    async def get_by_uri(self, uri: str) -> Source | None:
+        """Get a source by URI."""
+        raise NotImplementedError
+
+    async def list_by_type(
+        self, source_type: SourceType | None = None
+    ) -> Sequence[Source]:
+        """List sources by type."""
+        raise NotImplementedError
+
+    async def create_file(self, file: File) -> File:
+        """Create a new file record."""
+        raise NotImplementedError
+
+    async def upsert_author(self, author: Author) -> Author:
+        """Create a new author or return existing one if email already exists."""
+        raise NotImplementedError
+
+    async def upsert_author_file_mapping(
+        self, mapping: "AuthorFileMapping"
+    ) -> "AuthorFileMapping":
+        """Create a new author file mapping or return existing one if already exists."""
+        raise NotImplementedError
+
+
+class AuthorRepository(GenericRepository[Author]):
+    """Author repository with specific methods."""
+
+    async def get_by_name(self, name: str) -> Author | None:
+        """Get an author by name."""
+        raise NotImplementedError
+
+    async def get_by_email(self, email: str) -> Author | None:
+        """Get an author by email."""
+        raise NotImplementedError
+
+
+class SnippetRepository(GenericRepository[Snippet]):
+    """Snippet repository with specific methods."""
+
+    async def get_by_index(self, index_id: int) -> Sequence[Snippet]:
+        """Get all snippets for an index."""
+        raise NotImplementedError
+
+    async def delete_by_index(self, index_id: int) -> None:
+        """Delete all snippets for an index."""
+        raise NotImplementedError
+
+
+class FileRepository(GenericRepository[File]):
+    """File repository with specific methods."""
+
+    async def get_files_for_index(self, index_id: int) -> Sequence[File]:
+        """Get all files for an index."""
+        raise NotImplementedError

kodit/domain/services/__init__.py

@@ -0,0 +1 @@
+"""Domain services for business logic."""