kodit 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl

kodit/infrastructure/cloning/folder/factory.py

@@ -0,0 +1,119 @@
+"""Factory for creating folder-based working copies."""
+
+from pathlib import Path
+
+import structlog
+
+from kodit.domain.entities import AuthorFileMapping, Source, SourceType
+from kodit.domain.interfaces import NullProgressCallback, ProgressCallback
+from kodit.domain.repositories import SourceRepository
+from kodit.domain.value_objects import ProgressEvent
+from kodit.infrastructure.cloning.folder.working_copy import FolderWorkingCopyProvider
+from kodit.infrastructure.cloning.metadata import (
+    FolderFileMetadataExtractor,
+    NoOpAuthorExtractor,
+)
+
+
+class FolderSourceFactory:
+    """Factory for creating folder sources."""
+
+    def __init__(
+        self,
+        repository: SourceRepository,
+        working_copy: FolderWorkingCopyProvider,
+    ) -> None:
+        """Initialize the source factory."""
+        self.log = structlog.get_logger(__name__)
+        self.repository = repository
+        self.working_copy = working_copy
+        self.metadata_extractor = FolderFileMetadataExtractor()
+        self.author_extractor = NoOpAuthorExtractor()
+
+    async def create(
+        self, uri: str, progress_callback: ProgressCallback | None = None
+    ) -> Source:
+        """Create a folder source from a path."""
+        # Use null callback if none provided
+        if progress_callback is None:
+            progress_callback = NullProgressCallback()
+
+        directory = Path(uri).expanduser().resolve()
+
+        # Check if source already exists
+        source = await self.repository.get_by_uri(directory.as_uri())
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+            return source
+
+        # Validate directory exists
+        if not directory.exists():
+            msg = f"Folder does not exist: {directory}"
+            raise ValueError(msg)
+
+        # Prepare working copy
+        clone_path = await self.working_copy.prepare(directory.as_uri())
+
+        # Create source record
+        source = await self.repository.create_source(
+            Source(
+                uri=directory.as_uri(),
+                cloned_path=str(clone_path),
+                source_type=SourceType.FOLDER,
+            )
+        )
+
+        # Get all files to process
+        files = [f for f in clone_path.rglob("*") if f.is_file()]
+
+        # Process files
+        await self._process_files(source, files, progress_callback)
+
+        return source
+
+    async def _process_files(
+        self, source: Source, files: list[Path], progress_callback: ProgressCallback
+    ) -> None:
+        """Process files for a source."""
+        total_files = len(files)
+
+        # Notify start of operation
+        await progress_callback.on_progress(
+            ProgressEvent(
+                operation="process_files",
+                current=0,
+                total=total_files,
+                message="Processing files...",
+            )
+        )
+
+        for i, path in enumerate(files, 1):
+            if not path.is_file():
+                continue
+
+            # Extract file metadata
+            file_record = await self.metadata_extractor.extract(path, source)
+            await self.repository.create_file(file_record)
+
+            # Extract authors
+            authors = await self.author_extractor.extract(path, source)
+            for author in authors:
+                await self.repository.upsert_author_file_mapping(
+                    AuthorFileMapping(
+                        author_id=author.id,
+                        file_id=file_record.id,
+                    )
+                )
+
+            # Update progress
+            await progress_callback.on_progress(
+                ProgressEvent(
+                    operation="process_files",
+                    current=i,
+                    total=total_files,
+                    message=f"Processing {path.name}...",
+                )
+            )
+
+        # Notify completion
+        await progress_callback.on_complete("process_files")

kodit/infrastructure/cloning/folder/working_copy.py

@@ -0,0 +1,38 @@
+"""Working copy provider for folder-based sources."""
+
+import shutil
+from pathlib import Path
+
+
+class FolderWorkingCopyProvider:
+    """Working copy provider for folder-based sources."""
+
+    def __init__(self, clone_dir: Path) -> None:
+        """Initialize the provider."""
+        self.clone_dir = clone_dir
+
+    async def prepare(self, uri: str) -> Path:
+        """Prepare a folder working copy."""
+        # Handle file:// URIs
+        if uri.startswith("file://"):
+            from urllib.parse import urlparse
+
+            parsed = urlparse(uri)
+            directory = Path(parsed.path).expanduser().resolve()
+        else:
+            directory = Path(uri).expanduser().resolve()
+
+        # Clone into a local directory
+        clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
+        clone_path.mkdir(parents=True, exist_ok=True)
+
+        # Copy all files recursively, preserving directory structure, ignoring
+        # hidden files
+        shutil.copytree(
+            directory,
+            clone_path,
+            ignore=shutil.ignore_patterns(".*"),
+            dirs_exist_ok=True,
+        )
+
+        return clone_path

kodit/infrastructure/cloning/git/__init__.py

@@ -0,0 +1 @@
+"""Git cloning infrastructure."""

kodit/infrastructure/cloning/git/factory.py

@@ -0,0 +1,133 @@
+"""Factory for creating git-based working copies."""
+
+import tempfile
+from pathlib import Path
+
+import git
+import structlog
+
+from kodit.domain.entities import AuthorFileMapping, Source, SourceType
+from kodit.domain.interfaces import NullProgressCallback, ProgressCallback
+from kodit.domain.repositories import SourceRepository
+from kodit.domain.services.ignore_service import IgnoreService
+from kodit.domain.value_objects import ProgressEvent
+from kodit.infrastructure.cloning.git.working_copy import GitWorkingCopyProvider
+from kodit.infrastructure.cloning.metadata import (
+    GitAuthorExtractor,
+    GitFileMetadataExtractor,
+)
+from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
+
+
+class GitSourceFactory:
+    """Factory for creating git-based working copies."""
+
+    def __init__(
+        self,
+        repository: SourceRepository,
+        working_copy: GitWorkingCopyProvider,
+    ) -> None:
+        """Initialize the source factory."""
+        self.log = structlog.get_logger(__name__)
+        self.repository = repository
+        self.working_copy = working_copy
+        self.metadata_extractor = GitFileMetadataExtractor()
+        self.author_extractor = GitAuthorExtractor(repository)
+
+    async def create(
+        self, uri: str, progress_callback: ProgressCallback | None = None
+    ) -> Source:
+        """Create a git source from a URI."""
+        # Use null callback if none provided
+        if progress_callback is None:
+            progress_callback = NullProgressCallback()
+
+        # Normalize the URI
+        self.log.debug("Normalising git uri", uri=uri)
+        with tempfile.TemporaryDirectory() as temp_dir:
+            git.Repo.clone_from(uri, temp_dir)
+            remote = git.Repo(temp_dir).remote()
+            uri = remote.url
+
+        # Check if source already exists
+        self.log.debug("Checking if source already exists", uri=uri)
+        source = await self.repository.get_by_uri(uri)
+
+        if source:
+            self.log.info("Source already exists, reusing...", source_id=source.id)
+            return source
+
+        # Prepare working copy
+        clone_path = await self.working_copy.prepare(uri)
+
+        # Create source record
+        self.log.debug("Creating source", uri=uri, clone_path=str(clone_path))
+        source = await self.repository.create_source(
+            Source(
+                uri=uri,
+                cloned_path=str(clone_path),
+                source_type=SourceType.GIT,
+            )
+        )
+
+        # Get files to process using ignore patterns
+        ignore_provider = GitIgnorePatternProvider(clone_path)
+        ignore_service = IgnoreService(ignore_provider)
+        files = [
+            f
+            for f in clone_path.rglob("*")
+            if f.is_file() and not ignore_service.should_ignore(f)
+        ]
+
+        # Process files
+        self.log.info("Inspecting files", source_id=source.id, num_files=len(files))
+        await self._process_files(source, files, progress_callback)
+
+        return source
+
+    async def _process_files(
+        self, source: Source, files: list[Path], progress_callback: ProgressCallback
+    ) -> None:
+        """Process files for a source."""
+        total_files = len(files)
+
+        # Notify start of operation
+        await progress_callback.on_progress(
+            ProgressEvent(
+                operation="process_files",
+                current=0,
+                total=total_files,
+                message="Processing files...",
+            )
+        )
+
+        for i, path in enumerate(files, 1):
+            if not path.is_file():
+                continue
+
+            # Extract file metadata
+            file_record = await self.metadata_extractor.extract(path, source)
+            await self.repository.create_file(file_record)
+
+            # Extract authors
+            authors = await self.author_extractor.extract(path, source)
+            for author in authors:
+                await self.repository.upsert_author_file_mapping(
+                    AuthorFileMapping(
+                        author_id=author.id,
+                        file_id=file_record.id,
+                    )
+                )
+
+            # Update progress
+            await progress_callback.on_progress(
+                ProgressEvent(
+                    operation="process_files",
+                    current=i,
+                    total=total_files,
+                    message=f"Processing {path.name}...",
+                )
+            )
+
+        # Notify completion
+        await progress_callback.on_complete("process_files")

kodit/infrastructure/cloning/git/working_copy.py

@@ -0,0 +1,32 @@
+"""Working copy provider for git-based sources."""
+
+from pathlib import Path
+
+import git
+import structlog
+
+
+class GitWorkingCopyProvider:
+    """Working copy provider for git-based sources."""
+
+    def __init__(self, clone_dir: Path) -> None:
+        """Initialize the provider."""
+        self.clone_dir = clone_dir
+        self.log = structlog.get_logger(__name__)
+
+    async def prepare(self, uri: str) -> Path:
+        """Prepare a Git working copy."""
+        # Create a unique directory name for the clone
+        clone_path = self.clone_dir / uri.replace("/", "_").replace(":", "_")
+        clone_path.mkdir(parents=True, exist_ok=True)
+
+        try:
+            self.log.info("Cloning repository", uri=uri, clone_path=str(clone_path))
+            git.Repo.clone_from(uri, clone_path)
+        except git.GitCommandError as e:
+            if "already exists and is not an empty directory" not in str(e):
+                msg = f"Failed to clone repository: {e}"
+                raise ValueError(msg) from e
+            self.log.info("Repository already exists, reusing...", uri=uri)
+
+        return clone_path

kodit/infrastructure/cloning/metadata.py

@@ -0,0 +1,127 @@
+"""Metadata extraction for cloned sources."""
+
+import mimetypes
+from datetime import UTC, datetime
+from hashlib import sha256
+from pathlib import Path
+from typing import Any
+
+import aiofiles
+import git
+import structlog
+
+from kodit.domain.entities import Author, File, Source
+
+
+class BaseFileMetadataExtractor:
+    """Base class for file metadata extraction with common functionality."""
+
+    async def extract(self, path: Path, source: Source) -> File:
+        """Extract metadata from a file."""
+        # Get timestamps - to be implemented by subclasses
+        created_at, updated_at = await self._get_timestamps(path, source)
+
+        # Read file content and calculate metadata
+        async with aiofiles.open(path, "rb") as f:
+            content = await f.read()
+            mime_type = mimetypes.guess_type(path)
+            sha = sha256(content).hexdigest()
+
+            return File(
+                created_at=created_at,
+                updated_at=updated_at,
+                source_id=source.id,
+                cloned_path=str(path),
+                mime_type=mime_type[0]
+                if mime_type and mime_type[0]
+                else "application/octet-stream",
+                uri=path.as_uri(),
+                sha256=sha,
+                size_bytes=len(content),
+            )
+
+    async def _get_timestamps(
+        self, path: Path, source: Source
+    ) -> tuple[datetime, datetime]:
+        """Get creation and modification timestamps. To be implemented by subclasses."""
+        raise NotImplementedError
+
+
+class GitFileMetadataExtractor(BaseFileMetadataExtractor):
+    """Git-specific implementation for extracting file metadata."""
+
+    async def _get_timestamps(
+        self, path: Path, source: Source
+    ) -> tuple[datetime, datetime]:
+        """Get timestamps from Git history."""
+        git_repo = git.Repo(source.cloned_path)
+        commits = list(git_repo.iter_commits(paths=str(path), all=True))
+
+        if commits:
+            last_modified_at = commits[0].committed_datetime
+            first_modified_at = commits[-1].committed_datetime
+            return first_modified_at, last_modified_at
+        # Fallback to current time if no commits found
+        now = datetime.now(UTC)
+        return now, now
+
+
+class FolderFileMetadataExtractor(BaseFileMetadataExtractor):
+    """Folder-specific implementation for extracting file metadata."""
+
+    async def _get_timestamps(
+        self,
+        path: Path,
+        source: Source,  # noqa: ARG002
+    ) -> tuple[datetime, datetime]:
+        """Get timestamps from file system."""
+        stat = path.stat()
+        file_created_at = datetime.fromtimestamp(stat.st_ctime, UTC)
+        file_modified_at = datetime.fromtimestamp(stat.st_mtime, UTC)
+        return file_created_at, file_modified_at
+
+
+class GitAuthorExtractor:
+    """Author extractor for Git repositories."""
+
+    def __init__(self, repository: Any) -> None:
+        """Initialize the extractor."""
+        self.repository = repository
+        self.log = structlog.get_logger(__name__)
+
+    async def extract(self, path: Path, source: Source) -> list[Author]:
+        """Extract authors from a Git file."""
+        authors: list[Author] = []
+        git_repo = git.Repo(source.cloned_path)
+
+        try:
+            # Get the file's blame
+            blames = git_repo.blame("HEAD", str(path))
+
+            # Extract the blame's authors
+            actors = [
+                commit.author
+                for blame in blames or []
+                for commit in blame
+                if isinstance(commit, git.Commit)
+            ]
+
+            # Get or create the authors in the database
+            for actor in actors:
+                if actor.email:
+                    author = Author.from_actor(actor)
+                    author = await self.repository.upsert_author(author)
+                    authors.append(author)
+        except git.GitCommandError:
+            # Handle cases where file might not be tracked
+            pass
+
+        return authors
+
+
+class NoOpAuthorExtractor:
+    """No-op author extractor for sources that don't have author information."""
+
+    async def extract(self, path: Path, source: Source) -> list[Author]:  # noqa: ARG002
+        """Return empty list of authors."""
+        return []

kodit/infrastructure/embedding/__init__.py

@@ -0,0 +1 @@
+"""Embedding infrastructure module."""

kodit/infrastructure/embedding/embedding_factory.py

@@ -0,0 +1,87 @@
+"""Factory for creating embedding services with DDD architecture."""
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from kodit.config import AppContext, Endpoint
+from kodit.domain.entities import EmbeddingType
+from kodit.domain.services.embedding_service import EmbeddingDomainService
+from kodit.infrastructure.embedding.embedding_providers.local_embedding_provider import (  # noqa: E501
+    CODE,
+    LocalEmbeddingProvider,
+)
+from kodit.infrastructure.embedding.embedding_providers.openai_embedding_provider import (  # noqa: E501
+    OpenAIEmbeddingProvider,
+)
+from kodit.infrastructure.embedding.local_vector_search_repository import (
+    LocalVectorSearchRepository,
+)
+from kodit.infrastructure.embedding.vectorchord_vector_search_repository import (
+    TaskName,
+    VectorChordVectorSearchRepository,
+)
+from kodit.infrastructure.sqlalchemy.embedding_repository import (
+    SqlAlchemyEmbeddingRepository,
+)
+from kodit.log import log_event
+
+
+def _get_endpoint_configuration(app_context: AppContext) -> Endpoint | None:
+    """Get the endpoint configuration for the embedding service."""
+    return app_context.embedding_endpoint or app_context.default_endpoint or None
+
+
+def embedding_domain_service_factory(
+    task_name: TaskName, app_context: AppContext, session: AsyncSession
+) -> EmbeddingDomainService:
+    """Create an embedding domain service."""
+    # Create embedding repository
+    embedding_repository = SqlAlchemyEmbeddingRepository(session=session)
+
+    # Create embedding provider
+    endpoint = _get_endpoint_configuration(app_context)
+    if endpoint and endpoint.type == "openai":
+        log_event("kodit.embedding", {"provider": "openai"})
+        from openai import AsyncOpenAI
+
+        embedding_provider = OpenAIEmbeddingProvider(
+            openai_client=AsyncOpenAI(
+                api_key=endpoint.api_key or "default",
+                base_url=endpoint.base_url or "https://api.openai.com/v1",
+                timeout=10,
+                max_retries=2,
+            ),
+            model_name=endpoint.model or "text-embedding-3-small",
+        )
+    else:
+        log_event("kodit.embedding", {"provider": "local"})
+        embedding_provider = LocalEmbeddingProvider(CODE)
+
+    # Create vector search repository based on configuration
+    if app_context.default_search.provider == "vectorchord":
+        log_event("kodit.database", {"provider": "vectorchord"})
+        vector_search_repository = VectorChordVectorSearchRepository(
+            task_name, session, embedding_provider
+        )
+    elif app_context.default_search.provider == "sqlite":
+        log_event("kodit.database", {"provider": "sqlite"})
+        if task_name == "code":
+            embedding_type = EmbeddingType.CODE
+        elif task_name == "text":
+            embedding_type = EmbeddingType.TEXT
+        else:
+            raise ValueError(f"Invalid task name: {task_name}")
+
+        vector_search_repository = LocalVectorSearchRepository(
+            embedding_repository=embedding_repository,
+            embedding_provider=embedding_provider,
+            embedding_type=embedding_type,
+        )
+    else:
+        msg = f"Invalid semantic search provider: {app_context.default_search.provider}"
+        raise ValueError(msg)
+
+    # Create and return domain service
+    return EmbeddingDomainService(
+        embedding_provider=embedding_provider,
+        vector_search_repository=vector_search_repository,
+    )

kodit/infrastructure/embedding/embedding_providers/__init__.py

@@ -0,0 +1 @@
+"""Embedding providers module."""

kodit/infrastructure/embedding/embedding_providers/batching.py

@@ -0,0 +1,93 @@
+"""Utilities for batching embedding requests based on token counts and batch size.
+
+This module centralises the logic for splitting a list of ``EmbeddingRequest``
+into smaller sub-batches that respect token limits (and optionally a maximum
+number of items per batch).  Both the OpenAI and Local embedding providers use
+this functionality.
+"""
+
+from tiktoken import Encoding
+
+from kodit.domain.value_objects import EmbeddingRequest
+
+__all__ = [
+    "split_sub_batches",
+]
+
+
+DEFAULT_MAX_TOKENS = 8192  # A conservative upper-bound for most embedding models
+
+
+def split_sub_batches(
+    encoding: Encoding,
+    data: list[EmbeddingRequest],
+    *,
+    max_tokens: int = DEFAULT_MAX_TOKENS,
+    batch_size: int | None = None,
+) -> list[list[EmbeddingRequest]]:
+    """Split *data* into sub-batches constrained by tokens and size.
+
+    Parameters
+    ----------
+    encoding
+        A *tiktoken* ``Encoding`` instance capable of counting tokens.
+    data
+        List of :class:`kodit.domain.value_objects.EmbeddingRequest` objects.
+    max_tokens
+        Maximum number of tokens allowed in a single batch.  Defaults to
+        ``DEFAULT_MAX_TOKENS``.
+    batch_size
+        Optional maximum number of items per batch.  If *None*, no explicit
+        size constraint is applied (token limit still applies).
+
+    Returns
+    -------
+    list[list[EmbeddingRequest]]
+        A list of batches where each batch is a list of ``EmbeddingRequest``s.
+
+    """
+    batches: list[list[EmbeddingRequest]] = []
+    current_batch: list[EmbeddingRequest] = []
+    current_tokens = 0
+
+    for original_item in data:
+        # ------------------------------------------------------------------
+        # Ensure **individual** requests never exceed the token limit.
+        # If they do, we *truncate* them rather than sending an oversized
+        # request to the embedding model (which would raise a 400 error).
+        # ------------------------------------------------------------------
+
+        token_ids = encoding.encode(original_item.text, disallowed_special=())
+        if len(token_ids) > max_tokens:
+            # Keep only the first *max_tokens* tokens and decode back to text.
+            token_ids = token_ids[:max_tokens]
+            truncated_text = encoding.decode(token_ids)
+
+            # Create a *new* EmbeddingRequest to avoid mutating the caller's
+            # objects (side-effects can be surprising).
+            item = EmbeddingRequest(
+                snippet_id=original_item.snippet_id,
+                text=truncated_text,
+            )
+        else:
+            item = original_item
+
+        item_tokens = len(token_ids)
+
+        # Determine whether adding the item would violate limits for the
+        # *current* batch. Note: size constraint is optional.
+        token_overflow = current_tokens + item_tokens > max_tokens
+        size_overflow = batch_size is not None and len(current_batch) >= batch_size
+
+        if (token_overflow or size_overflow) and current_batch:
+            batches.append(current_batch)
+            current_batch = [item]
+            current_tokens = item_tokens
+        else:
+            current_batch.append(item)
+            current_tokens += item_tokens
+
+    if current_batch:
+        batches.append(current_batch)
+
+    return batches