kodit 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

kodit/domain/services/source_service.py

@@ -0,0 +1,85 @@
+"""Source service rewritten to work directly with AsyncSession."""
+
+from collections.abc import Callable
+from pathlib import Path
+
+import structlog
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from kodit.domain.entities import Source
+from kodit.domain.interfaces import ProgressCallback
+from kodit.domain.repositories import SourceRepository
+from kodit.infrastructure.cloning.folder.factory import FolderSourceFactory
+from kodit.infrastructure.cloning.folder.working_copy import FolderWorkingCopyProvider
+from kodit.infrastructure.cloning.git.factory import (
+    GitSourceFactory,
+    GitWorkingCopyProvider,
+)
+from kodit.infrastructure.git.git_utils import is_valid_clone_target
+from kodit.infrastructure.sqlalchemy.repository import SqlAlchemySourceRepository
+
+
+class SourceService:
+    """Source service."""
+
+    def __init__(
+        self,
+        clone_dir: Path,
+        session_factory: async_sessionmaker[AsyncSession] | Callable[[], AsyncSession],
+    ) -> None:
+        """Initialize the source service."""
+        self.clone_dir = clone_dir
+        self._session_factory = session_factory
+        self.log = structlog.get_logger(__name__)
+
+    async def get(self, source_id: int) -> Source:
+        """Get a source."""
+        async with self._session_factory() as session:
+            repo = SqlAlchemySourceRepository(session)
+
+            source = await repo.get(source_id)
+            if source is None:
+                raise ValueError(f"Source not found: {source_id}")
+
+            return source
+
+    async def create(
+        self, uri_or_path_like: str, progress_callback: ProgressCallback | None = None
+    ) -> Source:
+        """Create a source."""
+        async with self._session_factory() as session:
+            repo = SqlAlchemySourceRepository(session)
+            git_factory, folder_factory = self._build_factories(repo, session)
+
+            if is_valid_clone_target(uri_or_path_like):
+                source = await git_factory.create(uri_or_path_like, progress_callback)
+            elif Path(uri_or_path_like).is_dir():
+                source = await folder_factory.create(
+                    uri_or_path_like, progress_callback
+                )
+            else:
+                raise ValueError(f"Unsupported source: {uri_or_path_like}")
+
+            # Factories handle their own commits now
+            return source
+
+    def _build_factories(
+        self, repository: SourceRepository, session: AsyncSession
+    ) -> tuple[GitSourceFactory, FolderSourceFactory]:
+        # Git-specific collaborators
+        git_wc = GitWorkingCopyProvider(self.clone_dir)
+        git_factory = GitSourceFactory(
+            repository=repository,
+            working_copy=git_wc,
+            session=session,
+        )
+
+        # Folder-specific collaborators
+        fold_wc = FolderWorkingCopyProvider(self.clone_dir)
+        folder_factory = FolderSourceFactory(
+            repository=repository,
+            working_copy=fold_wc,
+            session=session,
+        )
+
+        return git_factory, folder_factory

kodit/domain/value_objects.py

@@ -0,0 +1,215 @@
+"""Domain value objects and DTOs."""
+
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+
+from kodit.domain.enums import SnippetExtractionStrategy
+
+
+class SearchType(Enum):
+    """Type of search to perform."""
+
+    BM25 = "bm25"
+    VECTOR = "vector"
+    HYBRID = "hybrid"
+
+
+@dataclass
+class SnippetExtractionRequest:
+    """Domain model for snippet extraction request."""
+
+    file_path: Path
+    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED
+
+
+@dataclass
+class SnippetExtractionResult:
+    """Domain model for snippet extraction result."""
+
+    snippets: list[str]
+    language: str
+
+
+@dataclass
+class Document:
+    """Generic document model for indexing."""
+
+    snippet_id: int
+    text: str
+
+
+@dataclass
+class SearchResult:
+    """Generic search result model."""
+
+    snippet_id: int
+    score: float
+
+
+@dataclass
+class IndexRequest:
+    """Generic indexing request."""
+
+    documents: list[Document]
+
+
+@dataclass
+class SimpleSearchRequest:
+    """Generic search request (single query string)."""
+
+    query: str
+    top_k: int = 10
+    search_type: SearchType = SearchType.BM25
+
+
+@dataclass
+class DeleteRequest:
+    """Generic deletion request."""
+
+    snippet_ids: list[int]
+
+
+@dataclass
+class IndexResult:
+    """Generic indexing result."""
+
+    snippet_id: int
+
+
+# Legacy aliases for backward compatibility
+BM25Document = Document
+BM25SearchResult = SearchResult
+BM25IndexRequest = IndexRequest
+BM25SearchRequest = SimpleSearchRequest
+BM25DeleteRequest = DeleteRequest
+
+VectorSearchRequest = Document
+VectorSearchResult = SearchResult
+VectorIndexRequest = IndexRequest
+VectorSearchQueryRequest = SimpleSearchRequest
+
+
+@dataclass
+class MultiSearchRequest:
+    """Domain model for multi-modal search request."""
+
+    top_k: int = 10
+    text_query: str | None = None
+    code_query: str | None = None
+    keywords: list[str] | None = None
+
+
+@dataclass
+class MultiSearchResult:
+    """Domain model for multi-modal search result."""
+
+    id: int
+    uri: str
+    content: str
+    original_scores: list[float]
+
+
+@dataclass
+class FusionRequest:
+    """Domain model for fusion request."""
+
+    id: int
+    score: float
+
+
+@dataclass
+class FusionResult:
+    """Domain model for fusion result."""
+
+    id: int
+    score: float
+    original_scores: list[float]
+
+
+@dataclass
+class IndexCreateRequest:
+    """Domain model for index creation request."""
+
+    source_id: int
+
+
+@dataclass
+class IndexRunRequest:
+    """Domain model for index run request."""
+
+    index_id: int
+
+
+@dataclass
+class ProgressEvent:
+    """Domain model for progress events."""
+
+    operation: str
+    current: int
+    total: int
+    message: str | None = None
+
+    @property
+    def percentage(self) -> float:
+        """Calculate the percentage of completion."""
+        return (self.current / self.total * 100) if self.total > 0 else 0.0
+
+
+@dataclass
+class EmbeddingRequest:
+    """Domain model for embedding request."""
+
+    snippet_id: int
+    text: str
+
+
+@dataclass
+class EmbeddingResponse:
+    """Domain model for embedding response."""
+
+    snippet_id: int
+    embedding: list[float]
+
+
+@dataclass
+class EnrichmentRequest:
+    """Domain model for enrichment request."""
+
+    snippet_id: int
+    text: str
+
+
+@dataclass
+class EnrichmentResponse:
+    """Domain model for enrichment response."""
+
+    snippet_id: int
+    text: str
+
+
+@dataclass
+class EnrichmentIndexRequest:
+    """Domain model for enrichment index request."""
+
+    requests: list[EnrichmentRequest]
+
+
+@dataclass
+class EnrichmentSearchRequest:
+    """Domain model for enrichment search request."""
+
+    query: str
+    top_k: int = 10
+
+
+@dataclass
+class IndexView:
+    """Domain model for index information."""
+
+    id: int
+    created_at: datetime
+    num_snippets: int
+    updated_at: datetime | None = None
+    source: str | None = None

kodit/infrastructure/__init__.py

@@ -0,0 +1 @@
+"""Infrastructure layer."""

kodit/infrastructure/bm25/__init__.py

@@ -0,0 +1 @@
+"""BM25 infrastructure module."""

kodit/infrastructure/bm25/bm25_factory.py

@@ -0,0 +1,28 @@
+"""Factory for creating BM25 repositories."""
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.config import AppContext
+from kodit.domain.services.bm25_service import BM25Repository
+from kodit.infrastructure.bm25.local_bm25_repository import LocalBM25Repository
+from kodit.infrastructure.bm25.vectorchord_bm25_repository import (
+    VectorChordBM25Repository,
+)
+
+
+def bm25_repository_factory(
+    app_context: AppContext, session: AsyncSession
+) -> BM25Repository:
+    """Create a BM25 repository based on configuration.
+
+    Args:
+        app_context: Application configuration context
+        session: SQLAlchemy async session
+
+    Returns:
+        BM25Repository instance
+
+    """
+    if app_context.default_search.provider == "vectorchord":
+        return VectorChordBM25Repository(session=session)
+    return LocalBM25Repository(data_dir=app_context.get_data_dir())

kodit/{bm25/local_bm25.py → infrastructure/bm25/local_bm25_repository.py}

@@ -1,4 +1,4 @@
-"""Locally hosted BM25 service primarily for use with SQLite."""
+"""Local BM25 repository implementation."""
 
 from __future__ import annotations
 
@@ -10,25 +10,31 @@ import aiofiles
 import Stemmer
 import structlog
 
-from kodit.bm25.keyword_search_service import (
-    BM25Document,
-    BM25Result,
-    KeywordSearchProvider,
+from kodit.domain.services.bm25_service import BM25Repository
+from kodit.domain.value_objects import (
+    BM25DeleteRequest,
+    BM25IndexRequest,
+    BM25SearchRequest,
+    BM25SearchResult,
 )
 
 if TYPE_CHECKING:
     import bm25s
     from bm25s.tokenization import Tokenized
 
-
 SNIPPET_IDS_FILE = "snippet_ids.jsonl"
 
 
-class BM25Service(KeywordSearchProvider):
-    """LocalBM25 service."""
+class LocalBM25Repository(BM25Repository):
+    """Local BM25 repository implementation."""
 
     def __init__(self, data_dir: Path) -> None:
-        """Initialize the BM25 service."""
+        """Initialize the local BM25 repository.
+
+        Args:
+            data_dir: Directory to store BM25 index files
+
+        """
         self.log = structlog.get_logger(__name__)
         self.index_path = data_dir / "bm25s_index"
         self.snippet_ids: list[int] = []
@@ -51,6 +57,7 @@ class BM25Service(KeywordSearchProvider):
         return self.__retriever
 
     def _tokenize(self, corpus: list[str]) -> list[list[str]] | Tokenized:
+        """Tokenize text corpus."""
         from bm25s import tokenize
 
         return tokenize(
@@ -61,23 +68,25 @@ class BM25Service(KeywordSearchProvider):
             show_progress=True,
         )
 
-    async def index(self, corpus: list[BM25Document]) -> None:
-        """Index a new corpus."""
+    async def index_documents(self, request: BM25IndexRequest) -> None:
+        """Index documents for BM25 search."""
         self.log.debug("Indexing corpus")
-        if not corpus or len(corpus) == 0:
+        if not request.documents:
             self.log.warning("Corpus is empty, skipping bm25 index")
             return
 
-        vocab = self._tokenize([doc.text for doc in corpus])
+        vocab = self._tokenize([doc.text for doc in request.documents])
         self._retriever().index(vocab, show_progress=False)
         self._retriever().save(self.index_path)
-        self.snippet_ids = self.snippet_ids + [doc.snippet_id for doc in corpus]
+        self.snippet_ids = self.snippet_ids + [
+            doc.snippet_id for doc in request.documents
+        ]
         async with aiofiles.open(self.index_path / SNIPPET_IDS_FILE, "w") as f:
             await f.write(json.dumps(self.snippet_ids))
 
-    async def retrieve(self, query: str, top_k: int = 2) -> list[BM25Result]:
-        """Retrieve from the index."""
-        if top_k == 0:
+    async def search(self, request: BM25SearchRequest) -> list[BM25SearchResult]:
+        """Search documents using BM25."""
+        if request.top_k == 0:
             self.log.warning("Top k is 0, returning empty list")
             return []
 
@@ -91,14 +100,14 @@ class BM25Service(KeywordSearchProvider):
             return []
 
         # Adjust top_k to not exceed corpus size
-        top_k = min(top_k, num_docs)
+        top_k = min(request.top_k, num_docs)
         self.log.debug(
             "Retrieving from index",
-            query=query,
+            query=request.query,
             top_k=top_k,
         )
 
-        query_tokens = self._tokenize([query])
+        query_tokens = self._tokenize([request.query])
 
         self.log.debug("Query tokens", query_tokens=query_tokens)
 
@@ -109,11 +118,13 @@ class BM25Service(KeywordSearchProvider):
         )
         self.log.debug("Raw results", results=results, scores=scores)
         return [
-            BM25Result(snippet_id=int(result), score=float(score))
+            BM25SearchResult(snippet_id=int(result), score=float(score))
             for result, score in zip(results[0], scores[0], strict=False)
             if score > 0.0
         ]
 
-    async def delete(self, snippet_ids: list[int]) -> None:  # noqa: ARG002
+    async def delete_documents(self, request: BM25DeleteRequest) -> None:
         """Delete documents from the index."""
+        # request parameter is unused as deletion is not supported
+        # ruff: noqa: ARG002
         self.log.warning("Deletion not supported for local BM25 index")

kodit/{bm25/vectorchord_bm25.py → infrastructure/bm25/vectorchord_bm25_repository.py}

@@ -1,4 +1,4 @@
-"""VectorChord repository for document operations."""
+"""VectorChord BM25 repository implementation."""
 
 from typing import Any
 
@@ -6,10 +6,12 @@ import structlog
 from sqlalchemy import Result, TextClause, bindparam, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from kodit.bm25.keyword_search_service import (
-    BM25Document,
-    BM25Result,
-    KeywordSearchProvider,
+from kodit.domain.services.bm25_service import BM25Repository
+from kodit.domain.value_objects import (
+    BM25DeleteRequest,
+    BM25IndexRequest,
+    BM25SearchRequest,
+    BM25SearchResult,
 )
 
 TABLE_NAME = "vectorchord_bm25_documents"
@@ -84,14 +86,16 @@ WHERE snippet_id IN :snippet_ids
 """  # noqa: S608
 
 
-class VectorChordBM25(KeywordSearchProvider):
-    """BM25 using VectorChord."""
+class VectorChordBM25Repository(BM25Repository):
+    """VectorChord BM25 repository implementation."""
 
-    def __init__(
-        self,
-        session: AsyncSession,
-    ) -> None:
-        """Initialize the VectorChord BM25."""
+    def __init__(self, session: AsyncSession) -> None:
+        """Initialize the VectorChord BM25 repository.
+
+        Args:
+            session: The SQLAlchemy async session to use for database operations
+
+        """
         self.__session = session
         self._initialized = False
         self.log = structlog.get_logger(__name__)
@@ -142,55 +146,56 @@ class VectorChordBM25(KeywordSearchProvider):
         """Commit the session."""
         await self.__session.commit()
 
-    async def index(self, corpus: list[BM25Document]) -> None:
-        """Index a new corpus."""
+    async def index_documents(self, request: BM25IndexRequest) -> None:
+        """Index documents for BM25 search."""
         # Filter out any documents that don't have a snippet_id or text
-        corpus = [
+        valid_documents = [
             doc
-            for doc in corpus
+            for doc in request.documents
             if doc.snippet_id is not None and doc.text is not None and doc.text != ""
         ]
 
-        if not corpus or len(corpus) == 0:
+        if not valid_documents:
             self.log.warning("Corpus is empty, skipping bm25 index")
             return
 
         # Execute inserts
         await self._execute(
             text(INSERT_QUERY),
-            [{"snippet_id": doc.snippet_id, "passage": doc.text} for doc in corpus],
+            [
+                {"snippet_id": doc.snippet_id, "passage": doc.text}
+                for doc in valid_documents
+            ],
         )
 
         # Tokenize the new documents with schema qualification
         await self._execute(text(UPDATE_QUERY))
         await self._commit()
 
-    async def delete(self, snippet_ids: list[int]) -> None:
-        """Delete documents from the index."""
-        await self._execute(
-            text(DELETE_QUERY).bindparams(bindparam("snippet_ids", expanding=True)),
-            {"snippet_ids": snippet_ids},
-        )
-        await self._commit()
-
-    async def retrieve(
-        self,
-        query: str,
-        top_k: int = 10,
-    ) -> list[BM25Result]:
-        """Search documents using BM25 similarity."""
-        if not query or query == "":
+    async def search(self, request: BM25SearchRequest) -> list[BM25SearchResult]:
+        """Search documents using BM25."""
+        if not request.query or request.query == "":
             return []
 
-        sql = text(SEARCH_QUERY).bindparams(query_text=query, limit=top_k)
+        sql = text(SEARCH_QUERY).bindparams(
+            query_text=request.query, limit=request.top_k
+        )
         try:
             result = await self._execute(sql)
             rows = result.mappings().all()
 
             return [
-                BM25Result(snippet_id=row["snippet_id"], score=row["bm25_score"])
+                BM25SearchResult(snippet_id=row["snippet_id"], score=row["bm25_score"])
                 for row in rows
             ]
         except Exception as e:
             msg = f"Error during BM25 search: {e}"
             raise RuntimeError(msg) from e
+
+    async def delete_documents(self, request: BM25DeleteRequest) -> None:
+        """Delete documents from the index."""
+        await self._execute(
+            text(DELETE_QUERY).bindparams(bindparam("snippet_ids", expanding=True)),
+            {"snippet_ids": request.snippet_ids},
+        )
+        await self._commit()

kodit/infrastructure/cloning/__init__.py

@@ -0,0 +1 @@
+"""Cloning infrastructure."""

kodit/infrastructure/cloning/folder/__init__.py

@@ -0,0 +1 @@
+"""Folder cloning infrastructure."""