kodit 0.0.1__py3-none-any.whl → 0.1.1__py3-none-any.whl

kodit/indexing/service.py

@@ -0,0 +1,153 @@
+"""Index service for managing code indexes.
+
+This module provides the IndexService class which handles the business logic for
+creating, listing, and running code indexes. It orchestrates the interaction between the
+file system, database operations (via IndexRepository), and provides a clean API for
+index management.
+"""
+
+from datetime import datetime
+
+import aiofiles
+import pydantic
+import structlog
+from tqdm.asyncio import tqdm
+
+from kodit.indexing.models import Snippet
+from kodit.indexing.repository import IndexRepository
+from kodit.sources.service import SourceService
+
+# List of MIME types that are supported for indexing and snippet creation
+MIME_WHITELIST = [
+    "text/plain",
+    "text/markdown",
+    "text/x-python",
+    "text/x-shellscript",
+    "text/x-sql",
+]
+
+
+class IndexView(pydantic.BaseModel):
+    """Data transfer object for index information.
+
+    This model represents the public interface for index data, providing a clean
+    view of index information without exposing internal implementation details.
+    """
+
+    id: int
+    created_at: datetime
+    updated_at: datetime | None = None
+    source_uri: str | None = None
+    num_snippets: int | None = None
+
+
+class IndexService:
+    """Service for managing code indexes.
+
+    This service handles the business logic for creating, listing, and running code
+    indexes. It coordinates between file system operations, database operations (via
+    IndexRepository), and provides a clean API for index management.
+    """
+
+    def __init__(
+        self, repository: IndexRepository, source_service: SourceService
+    ) -> None:
+        """Initialize the index service.
+
+        Args:
+            repository: The repository instance to use for database operations.
+            source_service: The source service instance to use for source validation.
+
+        """
+        self.repository = repository
+        self.source_service = source_service
+        self.log = structlog.get_logger(__name__)
+
+    async def create(self, source_id: int) -> IndexView:
+        """Create a new index for a source.
+
+        This method creates a new index for the specified source, after validating
+        that the source exists and doesn't already have an index.
+
+        Args:
+            source_id: The ID of the source to create an index for.
+
+        Returns:
+            An Index object representing the newly created index.
+
+        Raises:
+            ValueError: If the source doesn't exist or already has an index.
+
+        """
+        # Check if the source exists
+        source = await self.source_service.get(source_id)
+
+        index = await self.repository.create(source.id)
+        return IndexView(
+            id=index.id,
+            created_at=index.created_at,
+        )
+
+    async def list_indexes(self) -> list[IndexView]:
+        """List all available indexes with their details.
+
+        Returns:
+            A list of Index objects containing information about each index,
+            including file and snippet counts.
+
+        """
+        indexes = await self.repository.list_indexes()
+
+        # Transform database results into DTOs
+        return [
+            IndexView(
+                id=index.id,
+                created_at=index.created_at,
+                updated_at=index.updated_at,
+                num_snippets=await self.repository.num_snippets_for_index(index.id),
+            )
+            for index in indexes
+        ]
+
+    async def run(self, index_id: int) -> None:
+        """Run the indexing process for a specific index."""
+        # Get and validate index
+        index = await self.repository.get_by_id(index_id)
+        if not index:
+            msg = f"Index not found: {index_id}"
+            raise ValueError(msg)
+
+        # Create snippets for supported file types
+        await self._create_snippets(index_id)
+
+        # Update index timestamp
+        await self.repository.update_index_timestamp(index)
+
+    async def _create_snippets(
+        self,
+        index_id: int,
+    ) -> None:
+        """Create snippets for supported files.
+
+        Args:
+            index: The index to create snippets for.
+            file_list: List of files to create snippets from.
+            existing_snippets_set: Set of file IDs that already have snippets.
+
+        """
+        files = await self.repository.files_for_index(index_id)
+        for file in tqdm(files, total=len(files)):
+            # Skip unsupported file types
+            if file.mime_type not in MIME_WHITELIST:
+                self.log.debug("Skipping mime type", mime_type=file.mime_type)
+                continue
+
+            # Create snippet from file content
+            async with aiofiles.open(file.cloned_path, "rb") as f:
+                content = await f.read()
+                snippet = Snippet(
+                    index_id=index_id,
+                    file_id=file.id,
+                    content=content.decode("utf-8"),
+                )
+                await self.repository.add_snippet(snippet)

kodit/logging.py

@@ -93,6 +93,15 @@ def configure_logging(log_level: str, log_format: LogFormat) -> None:
         logging.getLogger(_log).handlers.clear()
         logging.getLogger(_log).propagate = True
 
+    # Configure SQLAlchemy loggers to use our structlog setup
+    for _log in ["sqlalchemy.engine", "alembic"]:
+        engine_logger = logging.getLogger(_log)
+        engine_logger.setLevel(logging.WARNING)  # Hide INFO logs by default
+        if log_level.upper() == "DEBUG":
+            engine_logger.setLevel(
+                logging.DEBUG
+            )  # Only show all logs when in DEBUG mode
+
     def handle_exception(
         exc_type: type[BaseException],
         exc_value: BaseException,

kodit/retreival/__init__.py

@@ -0,0 +1 @@
+"""Retrieval package for code search and retrieval functionality."""

kodit/retreival/repository.py

@@ -0,0 +1,76 @@
+"""Repository for retrieving code snippets and search results.
+
+This module provides the RetrievalRepository class which handles all database operations
+related to searching and retrieving code snippets, including string-based searches
+and their associated file information.
+"""
+
+from typing import TypeVar
+
+import pydantic
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.indexing.models import Snippet
+from kodit.sources.models import File
+
+T = TypeVar("T")
+
+
+class RetrievalResult(pydantic.BaseModel):
+    """Data transfer object for search results.
+
+    This model represents a single search result, containing both the file path
+    and the matching snippet content.
+    """
+
+    uri: str
+    content: str
+
+
+class RetrievalRepository:
+    """Repository for retrieving code snippets and search results.
+
+    This class provides methods for searching and retrieving code snippets from
+    the database, including string-based searches and their associated file information.
+    """
+
+    def __init__(self, session: AsyncSession) -> None:
+        """Initialize the retrieval repository.
+
+        Args:
+            session: The SQLAlchemy async session to use for database operations.
+
+        """
+        self.session = session
+
+    async def string_search(self, query: str) -> list[RetrievalResult]:
+        """Search for snippets containing the given query string.
+
+        This method performs a case-insensitive search for the query string within
+        snippet contents, returning up to 10 most recent matches.
+
+        Args:
+            query: The string to search for within snippet contents.
+
+        Returns:
+            A list of RetrievalResult objects containing the matching snippets
+            and their associated file paths.
+
+        """
+        search_query = (
+            select(Snippet, File)
+            .join(File, Snippet.file_id == File.id)
+            .where(Snippet.content.ilike(f"%{query}%"))
+            .limit(10)
+        )
+        rows = await self.session.execute(search_query)
+        results = list(rows.all())
+
+        return [
+            RetrievalResult(
+                uri=file.uri,
+                content=snippet.content,
+            )
+            for snippet, file in results
+        ]

kodit/retreival/service.py

@@ -0,0 +1,30 @@
+"""Retrieval service."""
+
+import pydantic
+
+from kodit.retreival.repository import RetrievalRepository, RetrievalResult
+
+
+class RetrievalRequest(pydantic.BaseModel):
+    """Request for a retrieval."""
+
+    query: str
+
+
+class Snippet(pydantic.BaseModel):
+    """Snippet model."""
+
+    content: str
+    file_path: str
+
+
+class RetrievalService:
+    """Service for retrieving relevant data."""
+
+    def __init__(self, repository: RetrievalRepository) -> None:
+        """Initialize the retrieval service."""
+        self.repository = repository
+
+    async def retrieve(self, request: RetrievalRequest) -> list[RetrievalResult]:
+        """Retrieve relevant data."""
+        return await self.repository.string_search(request.query)

kodit/sources/__init__.py

@@ -0,0 +1 @@
+"""Sources package for managing code source repositories and local directories."""

kodit/sources/models.py

@@ -0,0 +1,71 @@
+"""Source models for managing code sources.
+
+This module defines the SQLAlchemy models used for storing and managing code sources.
+It includes models for tracking different types of sources (git repositories and local
+folders) and their relationships.
+"""
+
+from sqlalchemy import ForeignKey, Integer, String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from kodit.database import Base, CommonMixin
+
+# Enable proper type hints for SQLAlchemy models
+__all__ = ["File", "Source"]
+
+
+class Source(Base, CommonMixin):
+    """Base model for tracking code sources.
+
+    This model serves as the parent table for different types of sources.
+    It provides common fields and relationships for all source types.
+
+    Attributes:
+        id: The unique identifier for the source.
+        created_at: Timestamp when the source was created.
+        updated_at: Timestamp when the source was last updated.
+        cloned_uri: A URI to a copy of the source on the local filesystem.
+        uri: The URI of the source.
+
+    """
+
+    __tablename__ = "sources"
+    uri: Mapped[str] = mapped_column(String(1024), index=True, unique=True)
+    cloned_path: Mapped[str] = mapped_column(String(1024))
+
+    def __init__(self, uri: str, cloned_path: str) -> None:
+        """Initialize a new Source instance for typing purposes."""
+        super().__init__()
+        self.uri = uri
+        self.cloned_path = cloned_path
+
+
+class File(Base, CommonMixin):
+    """File model."""
+
+    __tablename__ = "files"
+
+    source_id: Mapped[int] = mapped_column(ForeignKey("sources.id"))
+    mime_type: Mapped[str] = mapped_column(String(255), default="")
+    uri: Mapped[str] = mapped_column(String(1024), default="")
+    cloned_path: Mapped[str] = mapped_column(String(1024))
+    sha256: Mapped[str] = mapped_column(String(64), default="", index=True)
+    size_bytes: Mapped[int] = mapped_column(Integer, default=0)
+
+    def __init__(  # noqa: PLR0913
+        self,
+        source_id: int,
+        cloned_path: str,
+        mime_type: str = "",
+        uri: str = "",
+        sha256: str = "",
+        size_bytes: int = 0,
+    ) -> None:
+        """Initialize a new File instance for typing purposes."""
+        super().__init__()
+        self.source_id = source_id
+        self.cloned_path = cloned_path
+        self.mime_type = mime_type
+        self.uri = uri
+        self.sha256 = sha256
+        self.size_bytes = size_bytes

kodit/sources/repository.py

@@ -0,0 +1,110 @@
+"""Source repository for database operations.
+
+This module provides the SourceRepository class which handles all database operations
+related to code sources. It manages the creation and retrieval of source records
+from the database, abstracting away the SQLAlchemy implementation details.
+"""
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.sources.models import File, Source
+
+
+class SourceRepository:
+    """Repository for managing source database operations.
+
+    This class provides methods for creating and retrieving source records from the
+    database. It handles the low-level database operations and transaction management.
+
+    Args:
+        session: The SQLAlchemy async session to use for database operations.
+
+    """
+
+    def __init__(self, session: AsyncSession) -> None:
+        """Initialize the source repository."""
+        self.session = session
+
+    async def create_source(self, source: Source) -> Source:
+        """Create a new folder source record in the database.
+
+        This method creates both a Source record and a linked FolderSource record
+        in a single transaction.
+
+        Args:
+            path: The absolute path of the folder to create a source for.
+
+        Returns:
+            The created Source model instance.
+
+        Note:
+            This method commits the transaction to ensure the source.id is available
+            for creating the linked FolderSource record.
+
+        """
+        self.session.add(source)
+        await self.session.commit()
+        return source
+
+    async def create_file(self, file: File) -> File:
+        """Create a new file record in the database.
+
+        This method creates a new File record and adds it to the session.
+
+        """
+        self.session.add(file)
+        await self.session.commit()
+        return file
+
+    async def num_files_for_source(self, source_id: int) -> int:
+        """Get the number of files for a source.
+
+        Args:
+            source_id: The ID of the source to get the number of files for.
+
+        Returns:
+            The number of files for the source.
+
+        """
+        query = (
+            select(func.count()).select_from(File).where(File.source_id == source_id)
+        )
+        result = await self.session.execute(query)
+        return result.scalar_one()
+
+    async def list_sources(self) -> list[Source]:
+        """Retrieve all sources from the database.
+
+        Returns:
+            A list of Source instances.
+
+        """
+        query = select(Source).limit(10)
+        result = await self.session.execute(query)
+        return list(result.scalars())
+
+    async def get_source_by_uri(self, uri: str) -> Source | None:
+        """Get a source by its URI.
+
+        Args:
+            uri: The URI of the source to get.
+
+        Returns:
+            The source with the given URI, or None if it does not exist.
+
+        """
+        query = select(Source).where(Source.uri == uri)
+        result = await self.session.execute(query)
+        return result.scalar_one_or_none()
+
+    async def get_source_by_id(self, source_id: int) -> Source | None:
+        """Get a source by its ID.
+
+        Args:
+            source_id: The ID of the source to get.
+
+        """
+        query = select(Source).where(Source.id == source_id)
+        result = await self.session.execute(query)
+        return result.scalar_one_or_none()

kodit/sources/service.py

@@ -0,0 +1,208 @@
+"""Source service for managing code sources.
+
+This module provides the SourceService class which handles the business logic for
+creating and listing code sources. It orchestrates the interaction between the file
+system, database operations (via SourceRepository), and provides a clean API for
+source management.
+"""
+
+import mimetypes
+import shutil
+from datetime import datetime
+from hashlib import sha256
+from pathlib import Path
+
+import aiofiles
+import pydantic
+import structlog
+from tqdm import tqdm
+from uritools import isuri, urisplit
+
+from kodit.sources.models import File, Source
+from kodit.sources.repository import SourceRepository
+
+CLONE_DIR = Path(".kodit/clones").expanduser().resolve()
+
+
+class SourceView(pydantic.BaseModel):
+    """View model for displaying source information.
+
+    This model provides a clean interface for displaying source information,
+    containing only the essential fields needed for presentation.
+
+    Attributes:
+        id: The unique identifier for the source.
+        uri: The URI or path of the source.
+        created_at: Timestamp when the source was created.
+
+    """
+
+    id: int
+    uri: str
+    cloned_path: Path
+    created_at: datetime
+    num_files: int
+
+
+class SourceService:
+    """Service for managing code sources.
+
+    This service handles the business logic for creating and listing code sources.
+    It coordinates between file system operations, database operations (via
+    SourceRepository), and provides a clean API for source management.
+    """
+
+    def __init__(self, repository: SourceRepository) -> None:
+        """Initialize the source service.
+
+        Args:
+            repository: The repository instance to use for database operations.
+
+        """
+        self.repository = repository
+        self.log = structlog.get_logger(__name__)
+
+    async def get(self, source_id: int) -> SourceView:
+        """Get a source by ID.
+
+        Args:
+            source_id: The ID of the source to get.
+
+        """
+        source = await self.repository.get_source_by_id(source_id)
+        if not source:
+            msg = f"Source not found: {source_id}"
+            raise ValueError(msg)
+        return SourceView(
+            id=source.id,
+            uri=source.uri,
+            cloned_path=Path(source.cloned_path),
+            created_at=source.created_at,
+            num_files=await self.repository.num_files_for_source(source.id),
+        )
+
+    async def create(self, uri_or_path_like: str) -> SourceView:
+        """Create a new source from a URI.
+
+        Args:
+            uri: The URI of the source to create. Can be a git-like URI or a local
+                directory.
+
+        Raises:
+            ValueError: If the source type is not supported or if the folder doesn't
+                exist.
+
+        """
+        if Path(uri_or_path_like).is_dir():
+            return await self._create_folder_source(Path(uri_or_path_like))
+        if isuri(uri_or_path_like):
+            parsed = urisplit(uri_or_path_like)
+            if parsed.scheme == "file":
+                return await self._create_folder_source(Path(parsed.path))
+            msg = f"Unsupported source type: {uri_or_path_like}"
+            raise ValueError(msg)
+        msg = f"Unsupported source type: {uri_or_path_like}"
+        raise ValueError(msg)
+
+    async def _create_folder_source(self, directory: Path) -> SourceView:
+        """Create a folder source.
+
+        Args:
+            directory: The path to the local directory.
+
+        Raises:
+            ValueError: If the folder doesn't exist or is already added.
+
+        """
+        # Resolve the directory to an absolute path
+        directory = directory.expanduser().resolve()
+
+        # Check if the folder exists
+        if not directory.exists():
+            msg = f"Folder does not exist: {directory}"
+            raise ValueError(msg)
+
+        # Check if the folder is already added
+        if await self.repository.get_source_by_uri(directory.as_uri()):
+            msg = f"Directory already added: {directory}"
+            raise ValueError(msg)
+
+        # Clone into a local directory
+        clone_path = CLONE_DIR / directory.as_posix().replace("/", "_")
+        clone_path.mkdir(parents=True, exist_ok=True)
+
+        # Copy all files recursively, preserving directory structure, ignoring hidden
+        # files
+        shutil.copytree(
+            directory,
+            clone_path,
+            ignore=shutil.ignore_patterns(".*"),
+            dirs_exist_ok=True,
+        )
+
+        source = await self.repository.create_source(
+            Source(uri=directory.as_uri(), cloned_path=str(clone_path)),
+        )
+
+        # Add all files to the source
+        # Count total files for progress bar
+        file_count = sum(1 for _ in clone_path.rglob("*") if _.is_file())
+
+        # Process each file in the source directory
+        for path in tqdm(clone_path.rglob("*"), total=file_count):
+            await self._process_file(source.id, path.absolute())
+
+        return SourceView(
+            id=source.id,
+            uri=source.uri,
+            cloned_path=Path(source.cloned_path),
+            created_at=source.created_at,
+            num_files=await self.repository.num_files_for_source(source.id),
+        )
+
+    async def _process_file(
+        self,
+        source_id: int,
+        cloned_path: Path,
+    ) -> None:
+        """Process a single file for indexing."""
+        if not cloned_path.is_file():
+            return
+
+        async with aiofiles.open(cloned_path, "rb") as f:
+            content = await f.read()
+            mime_type = mimetypes.guess_type(cloned_path)
+            sha = sha256(content).hexdigest()
+
+            # Create file record
+            file = File(
+                source_id=source_id,
+                cloned_path=cloned_path.as_posix(),
+                mime_type=mime_type[0]
+                if mime_type and mime_type[0]
+                else "application/octet-stream",
+                uri=cloned_path.as_uri(),
+                sha256=sha,
+                size_bytes=len(content),
+            )
+
+            await self.repository.create_file(file)
+
+    async def list_sources(self) -> list[SourceView]:
+        """List all available sources.
+
+        Returns:
+            A list of SourceView objects containing information about each source.
+
+        """
+        sources = await self.repository.list_sources()
+        return [
+            SourceView(
+                id=source.id,
+                uri=source.uri,
+                cloned_path=Path(source.cloned_path),
+                created_at=source.created_at,
+                num_files=await self.repository.num_files_for_source(source.id),
+            )
+            for source in sources
+        ]