kodit 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

kodit/source/__init__.py

@@ -1 +0,0 @@
-"""Sources package for managing code source repositories and local directories."""

kodit/source/source_factories.py

@@ -1,356 +0,0 @@
-"""Source factories for creating different types of sources.
-
-This module provides factory classes for creating sources, improving cohesion by
-separating the concerns of different source types.
-"""
-
-import mimetypes
-import shutil
-import tempfile
-from abc import ABC, abstractmethod
-from datetime import UTC, datetime
-from hashlib import sha256
-from pathlib import Path
-from typing import Protocol
-
-import aiofiles
-import git
-import structlog
-from tqdm import tqdm
-
-from kodit.source.ignore import IgnorePatterns
-from kodit.source.source_models import (
-    Author,
-    AuthorFileMapping,
-    File,
-    Source,
-    SourceType,
-)
-from kodit.source.source_repository import SourceRepository
-
-
-class WorkingCopyProvider(Protocol):
-    """Protocol for providing working copies of sources."""
-
-    async def prepare(self, uri: str) -> Path:
-        """Prepare a working copy and return its path."""
-        ...
-
-
-class FileMetadataExtractor(Protocol):
-    """Protocol for extracting file metadata."""
-
-    async def extract(self, path: Path, source: Source) -> File:
-        """Extract metadata from a file."""
-        ...
-
-
-class AuthorExtractor(Protocol):
-    """Protocol for extracting author information."""
-
-    async def extract(self, path: Path, source: Source) -> list[Author]:
-        """Extract authors for a file."""
-        ...
-
-
-class SourceFactory(ABC):
-    """Abstract base class for source factories."""
-
-    def __init__(
-        self,
-        working_copy: WorkingCopyProvider,
-        metadata_extractor: FileMetadataExtractor,
-        author_extractor: AuthorExtractor,
-        repository: SourceRepository,
-    ) -> None:
-        """Initialize the source factory."""
-        self.working_copy = working_copy
-        self.metadata_extractor = metadata_extractor
-        self.author_extractor = author_extractor
-        self.repository = repository
-        self.log = structlog.get_logger(__name__)
-
-    @abstractmethod
-    async def create(self, uri: str) -> Source:
-        """Create a source from a URI."""
-        ...
-
-    async def _process_files(self, source: Source, files: list[Path]) -> None:
-        """Process files for a source."""
-        for path in tqdm(files, total=len(files), leave=False):
-            if not path.is_file():
-                continue
-
-            # Extract file metadata
-            file_record = await self.metadata_extractor.extract(path, source)
-            await self.repository.create_file(file_record)
-
-            # Extract authors
-            authors = await self.author_extractor.extract(path, source)
-            for author in authors:
-                await self.repository.upsert_author_file_mapping(
-                    AuthorFileMapping(
-                        author_id=author.id,
-                        file_id=file_record.id,
-                    )
-                )
-
-
-class GitSourceFactory(SourceFactory):
-    """Factory for creating Git sources."""
-
-    async def create(self, uri: str) -> Source:
-        """Create a git source from a URI."""
-        # Normalize the URI
-        self.log.debug("Normalising git uri", uri=uri)
-        with tempfile.TemporaryDirectory() as temp_dir:
-            git.Repo.clone_from(uri, temp_dir)
-            remote = git.Repo(temp_dir).remote()
-            uri = remote.url
-
-        # Check if source already exists
-        self.log.debug("Checking if source already exists", uri=uri)
-        source = await self.repository.get_source_by_uri(uri)
-
-        if source:
-            self.log.info("Source already exists, reusing...", source_id=source.id)
-            return source
-
-        # Prepare working copy
-        clone_path = await self.working_copy.prepare(uri)
-
-        # Create source record
-        self.log.debug("Creating source", uri=uri, clone_path=str(clone_path))
-        source = await self.repository.create_source(
-            Source(
-                uri=uri,
-                cloned_path=str(clone_path),
-                source_type=SourceType.GIT,
-            )
-        )
-
-        # Get files to process using ignore patterns
-        ignore_patterns = IgnorePatterns(clone_path)
-        files = [
-            f
-            for f in clone_path.rglob("*")
-            if f.is_file() and not ignore_patterns.should_ignore(f)
-        ]
-
-        # Process files
-        self.log.info("Inspecting files", source_id=source.id, num_files=len(files))
-        await self._process_files(source, files)
-
-        return source
-
-
-class FolderSourceFactory(SourceFactory):
-    """Factory for creating folder sources."""
-
-    async def create(self, uri: str) -> Source:
-        """Create a folder source from a path."""
-        directory = Path(uri).expanduser().resolve()
-
-        # Check if source already exists
-        source = await self.repository.get_source_by_uri(directory.as_uri())
-        if source:
-            self.log.info("Source already exists, reusing...", source_id=source.id)
-            return source
-
-        # Validate directory exists
-        if not directory.exists():
-            msg = f"Folder does not exist: {directory}"
-            raise ValueError(msg)
-
-        # Prepare working copy
-        clone_path = await self.working_copy.prepare(directory.as_uri())
-
-        # Create source record
-        source = await self.repository.create_source(
-            Source(
-                uri=directory.as_uri(),
-                cloned_path=str(clone_path),
-                source_type=SourceType.FOLDER,
-            )
-        )
-
-        # Get all files to process
-        files = [f for f in clone_path.rglob("*") if f.is_file()]
-
-        # Process files
-        await self._process_files(source, files)
-
-        return source
-
-
-class GitWorkingCopyProvider:
-    """Working copy provider for Git repositories."""
-
-    def __init__(self, clone_dir: Path) -> None:
-        """Initialize the provider."""
-        self.clone_dir = clone_dir
-        self.log = structlog.get_logger(__name__)
-
-    async def prepare(self, uri: str) -> Path:
-        """Prepare a Git working copy."""
-        # Create a unique directory name for the clone
-        clone_path = self.clone_dir / uri.replace("/", "_").replace(":", "_")
-        clone_path.mkdir(parents=True, exist_ok=True)
-
-        try:
-            self.log.info("Cloning repository", uri=uri, clone_path=str(clone_path))
-            git.Repo.clone_from(uri, clone_path)
-        except git.GitCommandError as e:
-            if "already exists and is not an empty directory" not in str(e):
-                msg = f"Failed to clone repository: {e}"
-                raise ValueError(msg) from e
-            self.log.info("Repository already exists, reusing...", uri=uri)
-
-        return clone_path
-
-
-class FolderWorkingCopyProvider:
-    """Working copy provider for local folders."""
-
-    def __init__(self, clone_dir: Path) -> None:
-        """Initialize the provider."""
-        self.clone_dir = clone_dir
-
-    async def prepare(self, uri: str) -> Path:
-        """Prepare a folder working copy."""
-        # Handle file:// URIs
-        if uri.startswith("file://"):
-            from urllib.parse import urlparse
-
-            parsed = urlparse(uri)
-            directory = Path(parsed.path).expanduser().resolve()
-        else:
-            directory = Path(uri).expanduser().resolve()
-
-        # Clone into a local directory
-        clone_path = self.clone_dir / directory.as_posix().replace("/", "_")
-        clone_path.mkdir(parents=True, exist_ok=True)
-
-        # Copy all files recursively, preserving directory structure, ignoring
-        # hidden files
-        shutil.copytree(
-            directory,
-            clone_path,
-            ignore=shutil.ignore_patterns(".*"),
-            dirs_exist_ok=True,
-        )
-
-        return clone_path
-
-
-class BaseFileMetadataExtractor:
-    """Base class for file metadata extraction with common functionality."""
-
-    async def extract(self, path: Path, source: Source) -> File:
-        """Extract metadata from a file."""
-        # Get timestamps - to be implemented by subclasses
-        created_at, updated_at = await self._get_timestamps(path, source)
-
-        # Read file content and calculate metadata
-        async with aiofiles.open(path, "rb") as f:
-            content = await f.read()
-            mime_type = mimetypes.guess_type(path)
-            sha = sha256(content).hexdigest()
-
-            return File(
-                created_at=created_at,
-                updated_at=updated_at,
-                source_id=source.id,
-                cloned_path=str(path),
-                mime_type=mime_type[0]
-                if mime_type and mime_type[0]
-                else "application/octet-stream",
-                uri=path.as_uri(),
-                sha256=sha,
-                size_bytes=len(content),
-            )
-
-    async def _get_timestamps(
-        self, path: Path, source: Source
-    ) -> tuple[datetime, datetime]:
-        """Get creation and modification timestamps. To be implemented by subclasses."""
-        raise NotImplementedError
-
-
-class GitFileMetadataExtractor(BaseFileMetadataExtractor):
-    """Git-specific implementation for extracting file metadata."""
-
-    async def _get_timestamps(
-        self, path: Path, source: Source
-    ) -> tuple[datetime, datetime]:
-        """Get timestamps from Git history."""
-        git_repo = git.Repo(source.cloned_path)
-        commits = list(git_repo.iter_commits(paths=str(path), all=True))
-
-        if commits:
-            last_modified_at = commits[0].committed_datetime
-            first_modified_at = commits[-1].committed_datetime
-            return first_modified_at, last_modified_at
-        # Fallback to current time if no commits found
-        now = datetime.now(UTC)
-        return now, now
-
-
-class FolderFileMetadataExtractor(BaseFileMetadataExtractor):
-    """Folder-specific implementation for extracting file metadata."""
-
-    async def _get_timestamps(
-        self,
-        path: Path,
-        source: Source,  # noqa: ARG002
-    ) -> tuple[datetime, datetime]:
-        """Get timestamps from file system."""
-        stat = path.stat()
-        file_created_at = datetime.fromtimestamp(stat.st_ctime, UTC)
-        file_modified_at = datetime.fromtimestamp(stat.st_mtime, UTC)
-        return file_created_at, file_modified_at
-
-
-class GitAuthorExtractor:
-    """Author extractor for Git repositories."""
-
-    def __init__(self, repository: SourceRepository) -> None:
-        """Initialize the extractor."""
-        self.repository = repository
-
-    async def extract(self, path: Path, source: Source) -> list[Author]:
-        """Extract authors from a Git file."""
-        authors: list[Author] = []
-        git_repo = git.Repo(source.cloned_path)
-
-        try:
-            # Get the file's blame
-            blames = git_repo.blame("HEAD", str(path))
-
-            # Extract the blame's authors
-            actors = [
-                commit.author
-                for blame in blames or []
-                for commit in blame
-                if isinstance(commit, git.Commit)
-            ]
-
-            # Get or create the authors in the database
-            for actor in actors:
-                if actor.email:
-                    author = Author.from_actor(actor)
-                    author = await self.repository.upsert_author(author)
-                    authors.append(author)
-        except git.GitCommandError:
-            # Handle cases where file might not be tracked
-            pass
-
-        return authors
-
-
-class NoOpAuthorExtractor:
-    """No-op author extractor for sources that don't have author information."""
-
-    async def extract(self, path: Path, source: Source) -> list[Author]:  # noqa: ARG002
-        """Return empty list of authors."""
-        return []

kodit/source/source_repository.py

@@ -1,169 +0,0 @@
-"""Source repository for database operations."""
-
-from sqlalchemy import func, select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from kodit.source.source_models import (
-    Author,
-    AuthorFileMapping,
-    File,
-    Source,
-    SourceType,
-)
-
-
-class SourceRepository:
-    """Repository for managing source database operations.
-
-    This class provides methods for creating and retrieving source records from the
-    database. It handles the low-level database operations and transaction management.
-
-    Args:
-        session: The SQLAlchemy async session to use for database operations.
-
-    """
-
-    def __init__(self, session: AsyncSession) -> None:
-        """Initialize the source repository."""
-        self.session = session
-
-    async def create_source(self, source: Source) -> Source:
-        """Add a new source to the database."""
-        # Validate the source
-        if source.type == SourceType.UNKNOWN:
-            msg = "Source type is required"
-            raise ValueError(msg)
-
-        self.session.add(source)
-        await self.session.commit()
-        return source
-
-    async def create_file(self, file: File) -> File:
-        """Create a new file record in the database.
-
-        This method creates a new File record and adds it to the session.
-
-        """
-        self.session.add(file)
-        await self.session.commit()
-        return file
-
-    async def list_files_for_source(self, source_id: int) -> list[File]:
-        """List all files for a source."""
-        query = select(File).where(File.source_id == source_id)
-        result = await self.session.execute(query)
-        return list(result.scalars())
-
-    async def num_files_for_source(self, source_id: int) -> int:
-        """Get the number of files for a source.
-
-        Args:
-            source_id: The ID of the source to get the number of files for.
-
-        Returns:
-            The number of files for the source.
-
-        """
-        query = (
-            select(func.count()).select_from(File).where(File.source_id == source_id)
-        )
-        result = await self.session.execute(query)
-        return result.scalar_one()
-
-    async def list_sources(self) -> list[Source]:
-        """Retrieve all sources from the database.
-
-        Returns:
-            A list of Source instances.
-
-        """
-        query = select(Source).limit(10)
-        result = await self.session.execute(query)
-        return list(result.scalars())
-
-    async def get_source_by_uri(self, uri: str) -> Source | None:
-        """Get a source by its URI.
-
-        Args:
-            uri: The URI of the source to get.
-
-        Returns:
-            The source with the given URI, or None if it does not exist.
-
-        """
-        query = select(Source).where(Source.uri == uri)
-        result = await self.session.execute(query)
-        return result.scalar_one_or_none()
-
-    async def get_source_by_id(self, source_id: int) -> Source | None:
-        """Get a source by its ID.
-
-        Args:
-            source_id: The ID of the source to get.
-
-        """
-        query = select(Source).where(Source.id == source_id)
-        result = await self.session.execute(query)
-        return result.scalar_one_or_none()
-
-    async def get_author_by_email(self, email: str) -> Author | None:
-        """Get an author by email."""
-        query = select(Author).where(Author.email == email)
-        result = await self.session.execute(query)
-        return result.scalar_one_or_none()
-
-    async def upsert_author(self, author: Author) -> Author:
-        """Create a new author or return existing one if email already exists.
-
-        Args:
-            author: The Author instance to upsert.
-
-        Returns:
-            The existing Author if one with the same email exists, otherwise the newly
-            created Author.
-
-        """
-        # First check if author already exists with same name and email
-        query = select(Author).where(
-            Author.name == author.name, Author.email == author.email
-        )
-        result = await self.session.execute(query)
-        existing_author = result.scalar_one_or_none()
-
-        if existing_author:
-            return existing_author
-
-        # Author doesn't exist, create new one
-        self.session.add(author)
-        await self.session.commit()
-        return author
-
-    async def upsert_author_file_mapping(
-        self, mapping: AuthorFileMapping
-    ) -> AuthorFileMapping:
-        """Create a new author file mapping or return existing one if already exists."""
-        # First check if mapping already exists with same author_id and file_id
-        query = select(AuthorFileMapping).where(
-            AuthorFileMapping.author_id == mapping.author_id,
-            AuthorFileMapping.file_id == mapping.file_id,
-        )
-        result = await self.session.execute(query)
-        existing_mapping = result.scalar_one_or_none()
-
-        if existing_mapping:
-            return existing_mapping
-
-        # Mapping doesn't exist, create new one
-        self.session.add(mapping)
-        await self.session.commit()
-        return mapping
-
-    async def list_files_for_author(self, author_id: int) -> list[File]:
-        """List all files for an author."""
-        query = (
-            select(File)
-            .join(AuthorFileMapping)
-            .where(AuthorFileMapping.author_id == author_id)
-        )
-        result = await self.session.execute(query)
-        return list(result.scalars())

kodit/source/source_service.py

@@ -1,150 +0,0 @@
-"""Source service for managing code sources.
-
-This module provides the SourceService class which handles the business logic for
-creating and listing code sources. It orchestrates the interaction between the file
-system, database operations (via SourceRepository), and provides a clean API for
-source management.
-"""
-
-from datetime import datetime
-from pathlib import Path
-
-import pydantic
-import structlog
-
-from kodit.source.git import is_valid_clone_target
-from kodit.source.source_factories import (
-    FolderFileMetadataExtractor,
-    FolderSourceFactory,
-    FolderWorkingCopyProvider,
-    GitAuthorExtractor,
-    GitFileMetadataExtractor,
-    GitSourceFactory,
-    GitWorkingCopyProvider,
-    NoOpAuthorExtractor,
-)
-from kodit.source.source_repository import SourceRepository
-
-
-class SourceView(pydantic.BaseModel):
-    """View model for displaying source information.
-
-    This model provides a clean interface for displaying source information,
-    containing only the essential fields needed for presentation.
-
-    Attributes:
-        id: The unique identifier for the source.
-        uri: The URI or path of the source.
-        created_at: Timestamp when the source was created.
-
-    """
-
-    id: int
-    uri: str
-    cloned_path: Path
-    created_at: datetime
-    num_files: int
-
-
-class SourceService:
-    """Service for managing code sources.
-
-    This service handles the business logic for creating and listing code sources.
-    It coordinates between file system operations, database operations (via
-    SourceRepository), and provides a clean API for source management.
-    """
-
-    def __init__(self, clone_dir: Path, repository: SourceRepository) -> None:
-        """Initialize the source service.
-
-        Args:
-            repository: The repository instance to use for database operations.
-
-        """
-        self.clone_dir = clone_dir
-        self.repository = repository
-        self.log = structlog.get_logger(__name__)
-
-        # Initialize factories
-        self._setup_factories()
-
-    def _setup_factories(self) -> None:
-        # Git-specific dependencies
-        git_working_copy = GitWorkingCopyProvider(self.clone_dir)
-        git_metadata_extractor = GitFileMetadataExtractor()
-        git_author_extractor = GitAuthorExtractor(self.repository)
-        self.git_factory = GitSourceFactory(
-            working_copy=git_working_copy,
-            metadata_extractor=git_metadata_extractor,
-            author_extractor=git_author_extractor,
-            repository=self.repository,
-        )
-
-        # Folder-specific dependencies
-        folder_working_copy = FolderWorkingCopyProvider(self.clone_dir)
-        folder_metadata_extractor = FolderFileMetadataExtractor()
-        no_op_author_extractor = NoOpAuthorExtractor()
-        self.folder_factory = FolderSourceFactory(
-            working_copy=folder_working_copy,
-            metadata_extractor=folder_metadata_extractor,
-            author_extractor=no_op_author_extractor,
-            repository=self.repository,
-        )
-
-    async def get(self, source_id: int) -> SourceView:
-        """Get a source by ID.
-
-        Args:
-            source_id: The ID of the source to get.
-
-        """
-        source = await self.repository.get_source_by_id(source_id)
-        if not source:
-            msg = f"Source not found: {source_id}"
-            raise ValueError(msg)
-        return SourceView(
-            id=source.id,
-            uri=source.uri,
-            cloned_path=Path(source.cloned_path),
-            created_at=source.created_at,
-            num_files=await self.repository.num_files_for_source(source.id),
-        )
-
-    async def create(self, uri_or_path_like: str) -> SourceView:
-        """Create a new source from a URI or path."""
-        # If it's possible to clone it, then do so
-        if is_valid_clone_target(uri_or_path_like):
-            source = await self.git_factory.create(uri_or_path_like)
-        # Otherwise just treat it as a directory
-        elif Path(uri_or_path_like).is_dir():
-            source = await self.folder_factory.create(uri_or_path_like)
-        else:
-            msg = f"Unsupported source: {uri_or_path_like}"
-            raise ValueError(msg)
-
-        return SourceView(
-            id=source.id,
-            uri=source.uri,
-            cloned_path=Path(source.cloned_path),
-            created_at=source.created_at,
-            num_files=await self.repository.num_files_for_source(source.id),
-        )
-
-    async def list_sources(self) -> list[SourceView]:
-        """List all available sources.
-
-        Returns:
-            A list of SourceView objects containing information about each source.
-
-        """
-        sources = await self.repository.list_sources()
-        return [
-            SourceView(
-                id=source.id,
-                uri=source.uri,
-                cloned_path=Path(source.cloned_path),
-                created_at=source.created_at,
-                num_files=await self.repository.num_files_for_source(source.id),
-            )
-            for source in sources
-        ]

kodit/util/__init__.py

@@ -1 +0,0 @@
-"""Utility functions and classes."""