kodit 0.3.2__py3-none-any.whl → 0.3.4__py3-none-any.whl

kodit/cli.py

@@ -19,12 +19,14 @@ from kodit.config import (
     with_session,
 )
 from kodit.domain.errors import EmptySourceError
-from kodit.domain.services.source_service import SourceService
+from kodit.domain.services.index_query_service import IndexQueryService
 from kodit.domain.value_objects import (
     MultiSearchRequest,
     MultiSearchResult,
     SnippetSearchFilters,
 )
+from kodit.infrastructure.indexing.fusion_service import ReciprocalRankFusionService
+from kodit.infrastructure.sqlalchemy.index_repository import SqlAlchemyIndexRepository
 from kodit.infrastructure.ui.progress import (
     create_lazy_progress_callback,
     create_multi_stage_progress_callback,
@@ -77,30 +79,28 @@ async def index(
 ) -> None:
     """List indexes, or index data sources."""
     log = structlog.get_logger(__name__)
-    source_service = SourceService(
-        clone_dir=app_context.get_clone_dir(),
-        session_factory=lambda: session,
-    )
     service = create_code_indexing_application_service(
         app_context=app_context,
         session=session,
-        source_service=source_service,
+    )
+    index_query_service = IndexQueryService(
+        index_repository=SqlAlchemyIndexRepository(session=session),
+        fusion_service=ReciprocalRankFusionService(),
     )
 
     if auto_index:
         log.info("Auto-indexing configuration", config=app_context.auto_indexing)
-        auto_sources = app_context.auto_indexing.sources
-        if not auto_sources:
+        if not app_context.auto_indexing or not app_context.auto_indexing.sources:
             click.echo("No auto-index sources configured.")
             return
-
+        auto_sources = app_context.auto_indexing.sources
         click.echo(f"Auto-indexing {len(auto_sources)} configured sources...")
         sources = [source.uri for source in auto_sources]
 
     if not sources:
         log_event("kodit.cli.index.list")
         # No source specified, list all indexes
-        indexes = await service.list_indexes()
+        indexes = await index_query_service.list_indexes()
         headers: list[str | Cell] = [
             "ID",
             "Created At",
@@ -113,8 +113,8 @@ async def index(
                 index.id,
                 index.created_at,
                 index.updated_at,
-                index.source,
-                index.num_snippets,
+                index.source.working_copy.remote_uri,
+                len(index.source.working_copy.files),
             ]
             for index in indexes
         ]
@@ -131,14 +131,12 @@ async def index(
 
         # Create a lazy progress callback that only shows progress when needed
         progress_callback = create_lazy_progress_callback()
-        s = await source_service.create(source, progress_callback)
-
-        index = await service.create_index(s.id)
+        index = await service.create_index_from_uri(source, progress_callback)
 
         # Create a new progress callback for the indexing operations
         indexing_progress_callback = create_multi_stage_progress_callback()
         try:
-            await service.run_index(index.id, indexing_progress_callback)
+            await service.run_index(index, indexing_progress_callback)
         except EmptySourceError as e:
             log.exception("Empty source error", error=e)
             msg = f"""{e}. This could mean:
@@ -243,14 +241,9 @@ async def code(  # noqa: PLR0913
     This works best if your query is code.
     """
     log_event("kodit.cli.search.code")
-    source_service = SourceService(
-        clone_dir=app_context.get_clone_dir(),
-        session_factory=lambda: session,
-    )
     service = create_code_indexing_application_service(
         app_context=app_context,
         session=session,
-        source_service=source_service,
     )
 
     filters = _parse_filters(
@@ -304,14 +297,9 @@ async def keyword(  # noqa: PLR0913
 ) -> None:
     """Search for snippets using keyword search."""
     log_event("kodit.cli.search.keyword")
-    source_service = SourceService(
-        clone_dir=app_context.get_clone_dir(),
-        session_factory=lambda: session,
-    )
     service = create_code_indexing_application_service(
         app_context=app_context,
         session=session,
-        source_service=source_service,
     )
 
     filters = _parse_filters(
@@ -368,14 +356,9 @@ async def text(  # noqa: PLR0913
     This works best if your query is text.
     """
     log_event("kodit.cli.search.text")
-    source_service = SourceService(
-        clone_dir=app_context.get_clone_dir(),
-        session_factory=lambda: session,
-    )
     service = create_code_indexing_application_service(
         app_context=app_context,
         session=session,
-        source_service=source_service,
     )
 
     filters = _parse_filters(
@@ -433,14 +416,9 @@ async def hybrid(  # noqa: PLR0913
 ) -> None:
     """Search for snippets using hybrid search."""
     log_event("kodit.cli.search.hybrid")
-    source_service = SourceService(
-        clone_dir=app_context.get_clone_dir(),
-        session_factory=lambda: session,
-    )
     service = create_code_indexing_application_service(
         app_context=app_context,
         session=session,
-        source_service=source_service,
     )
 
     # Parse keywords into a list of strings
@@ -490,14 +468,9 @@ async def snippets(
 ) -> None:
     """Show snippets with optional filtering by path or source."""
     log_event("kodit.cli.show.snippets")
-    source_service = SourceService(
-        clone_dir=app_context.get_clone_dir(),
-        session_factory=lambda: session,
-    )
     service = create_code_indexing_application_service(
         app_context=app_context,
         session=session,
-        source_service=source_service,
     )
     snippets = await service.list_snippets(file_path=by_path, source_uri=by_source)
     if output_format == "text":

kodit/domain/entities.py

@@ -1,200 +1,271 @@
-"""SQLAlchemy entities."""
-
-from datetime import UTC, datetime
-from enum import Enum
-
-from git import Actor
-from sqlalchemy import (
-    DateTime,
-    ForeignKey,
-    Integer,
-    String,
-    UnicodeText,
-    UniqueConstraint,
-)
-from sqlalchemy import Enum as SQLAlchemyEnum
-from sqlalchemy.ext.asyncio import AsyncAttrs
-from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
-from sqlalchemy.types import JSON
-
-
-class Base(AsyncAttrs, DeclarativeBase):
-    """Base class for all models."""
-
-
-class CommonMixin:
-    """Common mixin for all models."""
-
-    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), default=lambda: datetime.now(UTC)
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True),
-        default=lambda: datetime.now(UTC),
-        onupdate=lambda: datetime.now(UTC),
-    )
-
-
-class SourceType(Enum):
-    """The type of source."""
-
-    UNKNOWN = 0
-    FOLDER = 1
-    GIT = 2
-
-
-class Source(Base, CommonMixin):
-    """Base model for tracking code sources.
-
-    This model serves as the parent table for different types of sources.
-    It provides common fields and relationships for all source types.
-
-    Attributes:
-        id: The unique identifier for the source.
-        created_at: Timestamp when the source was created.
-        updated_at: Timestamp when the source was last updated.
-        cloned_uri: A URI to a copy of the source on the local filesystem.
-        uri: The URI of the source.
-
-    """
-
-    __tablename__ = "sources"
-    uri: Mapped[str] = mapped_column(String(1024), index=True, unique=True)
-    cloned_path: Mapped[str] = mapped_column(String(1024), index=True)
-    type: Mapped[SourceType] = mapped_column(
-        SQLAlchemyEnum(SourceType), default=SourceType.UNKNOWN, index=True
-    )
-
-    def __init__(self, uri: str, cloned_path: str, source_type: SourceType) -> None:
-        """Initialize a new Source instance for typing purposes."""
-        super().__init__()
-        self.uri = uri
-        self.cloned_path = cloned_path
-        self.type = source_type
-
-
-class Author(Base, CommonMixin):
-    """Author model."""
-
-    __tablename__ = "authors"
-
-    __table_args__ = (UniqueConstraint("name", "email", name="uix_author"),)
-
-    name: Mapped[str] = mapped_column(String(255), index=True)
-    email: Mapped[str] = mapped_column(String(255), index=True)
-
-    @staticmethod
-    def from_actor(actor: Actor) -> "Author":
-        """Create an Author from an Actor."""
-        return Author(name=actor.name, email=actor.email)
-
+"""Pure domain entities using Pydantic."""
 
-class AuthorFileMapping(Base, CommonMixin):
-    """Author file mapping model."""
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Protocol
+from urllib.parse import urlparse, urlunparse
 
-    __tablename__ = "author_file_mappings"
+from pydantic import AnyUrl, BaseModel
 
-    __table_args__ = (
-        UniqueConstraint("author_id", "file_id", name="uix_author_file_mapping"),
-    )
-
-    author_id: Mapped[int] = mapped_column(ForeignKey("authors.id"), index=True)
-    file_id: Mapped[int] = mapped_column(ForeignKey("files.id"), index=True)
-
-
-class File(Base, CommonMixin):
-    """File model."""
-
-    __tablename__ = "files"
-
-    source_id: Mapped[int] = mapped_column(ForeignKey("sources.id"))
-    mime_type: Mapped[str] = mapped_column(String(255), default="", index=True)
-    uri: Mapped[str] = mapped_column(String(1024), default="", index=True)
-    cloned_path: Mapped[str] = mapped_column(String(1024), index=True)
-    sha256: Mapped[str] = mapped_column(String(64), default="", index=True)
-    size_bytes: Mapped[int] = mapped_column(Integer, default=0)
-    extension: Mapped[str] = mapped_column(String(255), default="", index=True)
-
-    def __init__(  # noqa: PLR0913
-        self,
-        created_at: datetime,
-        updated_at: datetime,
-        source_id: int,
-        mime_type: str,
-        uri: str,
-        cloned_path: str,
-        sha256: str,
-        size_bytes: int,
-        extension: str,
-    ) -> None:
-        """Initialize a new File instance for typing purposes."""
-        super().__init__()
-        self.created_at = created_at
-        self.updated_at = updated_at
-        self.source_id = source_id
-        self.mime_type = mime_type
-        self.uri = uri
-        self.cloned_path = cloned_path
-        self.sha256 = sha256
-        self.size_bytes = size_bytes
-        self.extension = extension
-
-
-class EmbeddingType(Enum):
-    """Embedding type."""
-
-    CODE = 1
-    TEXT = 2
-
-
-class Embedding(Base, CommonMixin):
-    """Embedding model."""
-
-    __tablename__ = "embeddings"
-
-    snippet_id: Mapped[int] = mapped_column(ForeignKey("snippets.id"), index=True)
-    type: Mapped[EmbeddingType] = mapped_column(
-        SQLAlchemyEnum(EmbeddingType), index=True
-    )
-    embedding: Mapped[list[float]] = mapped_column(JSON)
-
-
-class Index(Base, CommonMixin):
-    """Index model."""
-
-    __tablename__ = "indexes"
-
-    source_id: Mapped[int] = mapped_column(
-        ForeignKey("sources.id"), unique=True, index=True
-    )
-
-    def __init__(self, source_id: int) -> None:
-        """Initialize the index."""
-        super().__init__()
-        self.source_id = source_id
-
-
-class Snippet(Base, CommonMixin):
-    """Snippet model."""
-
-    __tablename__ = "snippets"
-
-    file_id: Mapped[int] = mapped_column(ForeignKey("files.id"), index=True)
-    index_id: Mapped[int] = mapped_column(ForeignKey("indexes.id"), index=True)
-    content: Mapped[str] = mapped_column(UnicodeText, default="")
-    summary: Mapped[str] = mapped_column(UnicodeText, default="")
-
-    def __init__(
-        self,
-        file_id: int,
-        index_id: int,
-        content: str,
-        summary: str = "",
-    ) -> None:
-        """Initialize the snippet."""
-        super().__init__()
-        self.file_id = file_id
-        self.index_id = index_id
-        self.content = content
-        self.summary = summary
+from kodit.domain.value_objects import (
+    FileProcessingStatus,
+    SnippetContent,
+    SnippetContentType,
+    SourceType,
+)
+from kodit.utils.path_utils import path_from_uri
+
+
+class IgnorePatternProvider(Protocol):
+    """Protocol for ignore pattern providers."""
+
+    def should_ignore(self, path: Path) -> bool:
+        """Check if a path should be ignored."""
+        ...
+
+
+class Author(BaseModel):
+    """Author domain entity."""
+
+    id: int | None = None
+    name: str
+    email: str
+
+
+class File(BaseModel):
+    """File domain entity."""
+
+    id: int | None = None  # Is populated by repository
+    created_at: datetime | None = None  # Is populated by repository
+    updated_at: datetime | None = None  # Is populated by repository
+    uri: AnyUrl
+    sha256: str
+    authors: list[Author]
+    mime_type: str
+    file_processing_status: FileProcessingStatus
+
+    def as_path(self) -> Path:
+        """Return the file as a path."""
+        return path_from_uri(str(self.uri))
+
+    def extension(self) -> str:
+        """Return the file extension."""
+        return Path(self.as_path()).suffix.lstrip(".")
+
+
+class WorkingCopy(BaseModel):
+    """Working copy value object representing cloned source location."""
+
+    created_at: datetime | None = None  # Is populated by repository
+    updated_at: datetime | None = None  # Is populated by repository
+    remote_uri: AnyUrl
+    cloned_path: Path
+    source_type: SourceType
+    files: list[File]
+
+    @classmethod
+    def sanitize_local_path(cls, path: str) -> AnyUrl:
+        """Sanitize a local path."""
+        return AnyUrl(Path(path).resolve().absolute().as_uri())
+
+    @classmethod
+    def sanitize_git_url(cls, url: str) -> AnyUrl:
+        """Remove credentials from a git URL while preserving the rest of the URL.
+
+        This function handles various git URL formats:
+        - HTTPS URLs with username:password@host
+        - HTTPS URLs with username@host (no password)
+        - SSH URLs (left unchanged)
+        - File URLs (left unchanged)
+
+        Args:
+            url: The git URL that may contain credentials.
+
+        Returns:
+            The sanitized URL with credentials removed.
+
+        Examples:
+            >>> sanitize_git_url("https://phil:token@dev.azure.com/org/project/_git/repo")
+            "https://dev.azure.com/org/project/_git/repo"
+            >>> sanitize_git_url("https://username@github.com/user/repo.git")
+            "https://github.com/user/repo.git"
+            >>> sanitize_git_url("git@github.com:user/repo.git")
+            "ssh://git@github.com/user/repo.git"
+
+        """
+        # Handle SSH URLs (they don't have credentials in the URL format)
+        if url.startswith("git@"):
+            # Convert git@host:path to ssh://git@host/path format for AnyUrl
+            # This maintains the same semantic meaning while making it a valid URL
+            if ":" in url and not url.startswith("ssh://"):
+                host_path = url[4:]  # Remove "git@"
+                if ":" in host_path:
+                    host, path = host_path.split(":", 1)
+                    ssh_url = f"ssh://git@{host}/{path}"
+                    return AnyUrl(ssh_url)
+            return AnyUrl(url)
+        if url.startswith("ssh://"):
+            return AnyUrl(url)
+
+        # Handle file URLs
+        if url.startswith("file://"):
+            return AnyUrl(url)
+
+        try:
+            # Parse the URL
+            parsed = urlparse(url)
+
+            # If there are no credentials, return the URL as-is
+            if not parsed.username:
+                return AnyUrl(url)
+
+            # Reconstruct the URL without credentials
+            # scheme, netloc (without username/password), path, params, query, fragment
+            sanitized_netloc = parsed.hostname
+            if parsed.port:
+                sanitized_netloc = f"{parsed.hostname}:{parsed.port}"
+
+            return AnyUrl(
+                urlunparse(
+                    (
+                        parsed.scheme,
+                        sanitized_netloc,
+                        parsed.path,
+                        parsed.params,
+                        parsed.query,
+                        parsed.fragment,
+                    )
+                )
+            )
+
+        except Exception as e:
+            raise ValueError(f"Invalid URL: {url}") from e
+
+    def modified_or_deleted_files(self) -> list[File]:
+        """Return the modified or deleted files."""
+        return [
+            file
+            for file in self.files
+            if file.file_processing_status
+            in (FileProcessingStatus.MODIFIED, FileProcessingStatus.DELETED)
+        ]
+
+    def list_filesystem_paths(
+        self, ignore_provider: IgnorePatternProvider
+    ) -> list[Path]:
+        """List the filesystem paths of the files in the working copy."""
+        if not self.cloned_path.exists():
+            raise ValueError(f"Cloned path does not exist: {self.cloned_path}")
+
+        return [
+            f
+            for f in self.cloned_path.rglob("*")
+            if f.is_file() and not ignore_provider.should_ignore(f)
+        ]
+
+    def dirty_files(self) -> list[File]:
+        """Return the dirty files."""
+        return [
+            file
+            for file in self.files
+            if file.file_processing_status
+            in (FileProcessingStatus.MODIFIED, FileProcessingStatus.ADDED)
+        ]
+
+    def changed_files(self) -> list[File]:
+        """Return the changed files."""
+        return [
+            file
+            for file in self.files
+            if file.file_processing_status != FileProcessingStatus.CLEAN
+        ]
+
+    def clear_file_processing_statuses(self) -> None:
+        """Clear the file processing statuses."""
+        # First remove any files that are marked for deletion
+        self.files = [
+            file
+            for file in self.files
+            if file.file_processing_status != FileProcessingStatus.DELETED
+        ]
+        # Then clear the statuses for the remaining files
+        for file in self.files:
+            file.file_processing_status = FileProcessingStatus.CLEAN
+
+
+class Source(BaseModel):
+    """Source domain entity."""
+
+    id: int | None = None  # Is populated by repository
+    created_at: datetime | None = None  # Is populated by repository
+    updated_at: datetime | None = None  # Is populated by repository
+    working_copy: WorkingCopy
+
+
+class Snippet(BaseModel):
+    """Snippet domain entity."""
+
+    id: int | None = None  # Is populated by repository
+    created_at: datetime | None = None  # Is populated by repository
+    updated_at: datetime | None = None  # Is populated by repository
+    derives_from: list[File]
+    original_content: SnippetContent | None = None
+    summary_content: SnippetContent | None = None
+
+    def original_text(self) -> str:
+        """Return the original content of the snippet."""
+        if self.original_content is None:
+            return ""
+        return self.original_content.value
+
+    def summary_text(self) -> str:
+        """Return the summary content of the snippet."""
+        if self.summary_content is None:
+            return ""
+        return self.summary_content.value
+
+    def add_original_content(self, content: str, language: str) -> None:
+        """Add an original content to the snippet."""
+        self.original_content = SnippetContent(
+            type=SnippetContentType.ORIGINAL,
+            value=content,
+            language=language,
+        )
+
+    def add_summary(self, summary: str) -> None:
+        """Add a summary to the snippet."""
+        self.summary_content = SnippetContent(
+            type=SnippetContentType.SUMMARY,
+            value=summary,
+            language="markdown",
+        )
+
+
+class Index(BaseModel):
+    """Index domain entity."""
+
+    id: int
+    created_at: datetime
+    updated_at: datetime
+    source: Source
+    snippets: list[Snippet]
+
+    def delete_snippets_for_files(self, files: list[File]) -> None:
+        """Delete the snippets that derive from a list of files."""
+        self.snippets = [
+            snippet
+            for snippet in self.snippets
+            if not any(file in snippet.derives_from for file in files)
+        ]
+
+
+# FUTURE: Remove this type, use the domain to get the required information.
+@dataclass(frozen=True)
+class SnippetWithContext:
+    """Domain model for snippet with associated context information."""
+
+    source: Source
+    file: File
+    authors: list[Author]
+    snippet: Snippet

kodit/domain/protocols.py

@@ -0,0 +1,61 @@
+"""Repository protocol interfaces for the domain layer."""
+
+from collections.abc import Sequence
+from typing import Protocol
+
+from pydantic import AnyUrl
+
+from kodit.domain.entities import Index, Snippet, SnippetWithContext, WorkingCopy
+from kodit.domain.value_objects import MultiSearchRequest
+
+
+class IndexRepository(Protocol):
+    """Repository interface for Index entities."""
+
+    async def create(self, uri: AnyUrl, working_copy: WorkingCopy) -> Index:
+        """Create an index for a source."""
+        ...
+
+    async def update(self, index: Index) -> None:
+        """Update an index."""
+        ...
+
+    async def get(self, index_id: int) -> Index | None:
+        """Get an index by ID."""
+        ...
+
+    async def all(self) -> list[Index]:
+        """List all indexes."""
+        ...
+
+    async def get_by_uri(self, uri: AnyUrl) -> Index | None:
+        """Get an index by source URI."""
+        ...
+
+    async def update_index_timestamp(self, index_id: int) -> None:
+        """Update the timestamp of an index."""
+        ...
+
+    async def add_snippets(self, index_id: int, snippets: list[Snippet]) -> None:
+        """Add snippets to an index."""
+        ...
+
+    async def update_snippets(self, index_id: int, snippets: list[Snippet]) -> None:
+        """Update snippets for an index."""
+        ...
+
+    async def delete_snippets(self, index_id: int) -> None:
+        """Delete all snippets from an index."""
+        ...
+
+    async def delete_snippets_by_file_ids(self, file_ids: list[int]) -> None:
+        """Delete snippets by file IDs."""
+        ...
+
+    async def search(self, request: MultiSearchRequest) -> Sequence[SnippetWithContext]:
+        """Search snippets with filters."""
+        ...
+
+    async def get_snippets_by_ids(self, ids: list[int]) -> list[SnippetWithContext]:
+        """Get snippets by their IDs."""
+        ...