kodit 0.4.2__py3-none-any.whl → 0.5.0__py3-none-any.whl

kodit/domain/entities.py

@@ -1,323 +0,0 @@
-"""Pure domain entities using Pydantic."""
-
-import shutil
-from dataclasses import dataclass
-from datetime import datetime
-from pathlib import Path
-from typing import Any, Protocol
-from urllib.parse import urlparse, urlunparse
-
-from pydantic import AnyUrl, BaseModel
-
-from kodit.domain.value_objects import (
-    FileProcessingStatus,
-    QueuePriority,
-    SnippetContent,
-    SnippetContentType,
-    SourceType,
-    TaskType,
-)
-from kodit.utils.path_utils import path_from_uri
-
-
-class IgnorePatternProvider(Protocol):
-    """Protocol for ignore pattern providers."""
-
-    def should_ignore(self, path: Path) -> bool:
-        """Check if a path should be ignored."""
-        ...
-
-
-class Author(BaseModel):
-    """Author domain entity."""
-
-    id: int | None = None
-    name: str
-    email: str
-
-
-class File(BaseModel):
-    """File domain entity."""
-
-    id: int | None = None  # Is populated by repository
-    created_at: datetime | None = None  # Is populated by repository
-    updated_at: datetime | None = None  # Is populated by repository
-    uri: AnyUrl
-    sha256: str
-    authors: list[Author]
-    mime_type: str
-    file_processing_status: FileProcessingStatus
-
-    def as_path(self) -> Path:
-        """Return the file as a path."""
-        return path_from_uri(str(self.uri))
-
-    def extension(self) -> str:
-        """Return the file extension."""
-        return Path(self.as_path()).suffix.lstrip(".")
-
-
-class WorkingCopy(BaseModel):
-    """Working copy value object representing cloned source location."""
-
-    created_at: datetime | None = None  # Is populated by repository
-    updated_at: datetime | None = None  # Is populated by repository
-    remote_uri: AnyUrl
-    cloned_path: Path
-    source_type: SourceType
-    files: list[File]
-
-    @classmethod
-    def sanitize_local_path(cls, path: str) -> AnyUrl:
-        """Sanitize a local path."""
-        return AnyUrl(Path(path).resolve().absolute().as_uri())
-
-    @classmethod
-    def sanitize_git_url(cls, url: str) -> AnyUrl:
-        """Remove credentials from a git URL while preserving the rest of the URL.
-
-        This function handles various git URL formats:
-        - HTTPS URLs with username:password@host
-        - HTTPS URLs with username@host (no password)
-        - SSH URLs (left unchanged)
-        - File URLs (left unchanged)
-
-        Args:
-            url: The git URL that may contain credentials.
-
-        Returns:
-            The sanitized URL with credentials removed.
-
-        Examples:
-            >>> sanitize_git_url("https://phil:token@dev.azure.com/org/project/_git/repo")
-            "https://dev.azure.com/org/project/_git/repo"
-            >>> sanitize_git_url("https://username@github.com/user/repo.git")
-            "https://github.com/user/repo.git"
-            >>> sanitize_git_url("git@github.com:user/repo.git")
-            "ssh://git@github.com/user/repo.git"
-
-        """
-        # Handle SSH URLs (they don't have credentials in the URL format)
-        if url.startswith("git@"):
-            # Convert git@host:path to ssh://git@host/path format for AnyUrl
-            # This maintains the same semantic meaning while making it a valid URL
-            if ":" in url and not url.startswith("ssh://"):
-                host_path = url[4:]  # Remove "git@"
-                if ":" in host_path:
-                    host, path = host_path.split(":", 1)
-                    ssh_url = f"ssh://git@{host}/{path}"
-                    return AnyUrl(ssh_url)
-            return AnyUrl(url)
-        if url.startswith("ssh://"):
-            return AnyUrl(url)
-
-        # Handle file URLs
-        if url.startswith("file://"):
-            return AnyUrl(url)
-
-        try:
-            # Parse the URL
-            parsed = urlparse(url)
-
-            # If there are no credentials, return the URL as-is
-            if not parsed.username:
-                return AnyUrl(url)
-
-            # Reconstruct the URL without credentials
-            # scheme, netloc (without username/password), path, params, query, fragment
-            sanitized_netloc = parsed.hostname
-            if parsed.port:
-                sanitized_netloc = f"{parsed.hostname}:{parsed.port}"
-
-            return AnyUrl(
-                urlunparse(
-                    (
-                        parsed.scheme,
-                        sanitized_netloc,
-                        parsed.path,
-                        parsed.params,
-                        parsed.query,
-                        parsed.fragment,
-                    )
-                )
-            )
-
-        except Exception as e:
-            raise ValueError(f"Invalid URL: {url}") from e
-
-    def modified_or_deleted_files(self) -> list[File]:
-        """Return the modified or deleted files."""
-        return [
-            file
-            for file in self.files
-            if file.file_processing_status
-            in (FileProcessingStatus.MODIFIED, FileProcessingStatus.DELETED)
-        ]
-
-    def list_filesystem_paths(
-        self, ignore_provider: IgnorePatternProvider
-    ) -> list[Path]:
-        """List the filesystem paths of the files in the working copy."""
-        if not self.cloned_path.exists():
-            raise ValueError(f"Cloned path does not exist: {self.cloned_path}")
-
-        return [
-            f
-            for f in self.cloned_path.rglob("*")
-            if f.is_file() and not ignore_provider.should_ignore(f)
-        ]
-
-    def dirty_files(self) -> list[File]:
-        """Return the dirty files."""
-        return [
-            file
-            for file in self.files
-            if file.file_processing_status
-            in (FileProcessingStatus.MODIFIED, FileProcessingStatus.ADDED)
-        ]
-
-    def changed_files(self) -> list[File]:
-        """Return the changed files."""
-        return [
-            file
-            for file in self.files
-            if file.file_processing_status != FileProcessingStatus.CLEAN
-        ]
-
-    def clear_file_processing_statuses(self) -> None:
-        """Clear the file processing statuses."""
-        # First remove any files that are marked for deletion
-        self.files = [
-            file
-            for file in self.files
-            if file.file_processing_status != FileProcessingStatus.DELETED
-        ]
-        # Then clear the statuses for the remaining files
-        for file in self.files:
-            file.file_processing_status = FileProcessingStatus.CLEAN
-
-    def delete(self) -> None:
-        """Delete the working copy."""
-        shutil.rmtree(self.cloned_path)
-
-
-class Source(BaseModel):
-    """Source domain entity."""
-
-    id: int | None = None  # Is populated by repository
-    created_at: datetime | None = None  # Is populated by repository
-    updated_at: datetime | None = None  # Is populated by repository
-    working_copy: WorkingCopy
-
-
-class Snippet(BaseModel):
-    """Snippet domain entity."""
-
-    id: int | None = None  # Is populated by repository
-    created_at: datetime | None = None  # Is populated by repository
-    updated_at: datetime | None = None  # Is populated by repository
-    derives_from: list[File]
-    original_content: SnippetContent | None = None
-    summary_content: SnippetContent | None = None
-
-    def original_text(self) -> str:
-        """Return the original content of the snippet."""
-        if self.original_content is None:
-            return ""
-        return self.original_content.value
-
-    def summary_text(self) -> str:
-        """Return the summary content of the snippet."""
-        if self.summary_content is None:
-            return ""
-        return self.summary_content.value
-
-    def add_original_content(self, content: str, language: str) -> None:
-        """Add an original content to the snippet."""
-        self.original_content = SnippetContent(
-            type=SnippetContentType.ORIGINAL,
-            value=content,
-            language=language,
-        )
-
-    def add_summary(self, summary: str) -> None:
-        """Add a summary to the snippet."""
-        self.summary_content = SnippetContent(
-            type=SnippetContentType.SUMMARY,
-            value=summary,
-            language="markdown",
-        )
-
-
-class Index(BaseModel):
-    """Index domain entity."""
-
-    id: int
-    created_at: datetime
-    updated_at: datetime
-    source: Source
-    snippets: list[Snippet]
-
-    def delete_snippets_for_files(self, files: list[File]) -> None:
-        """Delete the snippets that derive from a list of files."""
-        self.snippets = [
-            snippet
-            for snippet in self.snippets
-            if not any(file in snippet.derives_from for file in files)
-        ]
-
-
-# FUTURE: Remove this type, use the domain to get the required information.
-@dataclass(frozen=True)
-class SnippetWithContext:
-    """Domain model for snippet with associated context information."""
-
-    source: Source
-    file: File
-    authors: list[Author]
-    snippet: Snippet
-
-
-class Task(BaseModel):
-    """Represents an item in the queue waiting to be processed.
-
-    If the item exists, that means it is in the queue and waiting to be processed. There
-    is no status associated.
-    """
-
-    id: str  # Is a unique key to deduplicate items in the queue
-    type: TaskType  # Task type
-    priority: int  # Priority (higher number = higher priority)
-    payload: dict[str, Any]  # Task-specific data
-
-    created_at: datetime | None = None  # Is populated by repository
-    updated_at: datetime | None = None  # Is populated by repository
-
-    @staticmethod
-    def create(task_type: TaskType, priority: int, payload: dict[str, Any]) -> "Task":
-        """Create a task."""
-        return Task(
-            id=Task._create_id(task_type, payload),
-            type=task_type,
-            priority=priority,
-            payload=payload,
-        )
-
-    @staticmethod
-    def _create_id(task_type: TaskType, payload: dict[str, Any]) -> str:
-        """Create a unique id for a task."""
-        if task_type == TaskType.INDEX_UPDATE:
-            return str(payload["index_id"])
-
-        raise ValueError(f"Unknown task type: {task_type}")
-
-    @staticmethod
-    def create_index_update_task(
-        index_id: int, priority: QueuePriority = QueuePriority.USER_INITIATED
-    ) -> "Task":
-        """Create an index update task."""
-        return Task.create(
-            task_type=TaskType.INDEX_UPDATE,
-            priority=priority.value,
-            payload={"index_id": index_id},
-        )

kodit/domain/services/index_query_service.py

@@ -1,70 +0,0 @@
-"""Index query service."""
-
-from abc import ABC, abstractmethod
-
-from kodit.domain.entities import Index, SnippetWithContext
-from kodit.domain.protocols import IndexRepository
-from kodit.domain.value_objects import (
-    FusionRequest,
-    FusionResult,
-    MultiSearchRequest,
-)
-
-
-class FusionService(ABC):
-    """Abstract fusion service interface."""
-
-    @abstractmethod
-    def reciprocal_rank_fusion(
-        self, rankings: list[list[FusionRequest]], k: float = 60
-    ) -> list[FusionResult]:
-        """Perform reciprocal rank fusion on search results."""
-
-
-class IndexQueryService:
-    """Index query service."""
-
-    def __init__(
-        self,
-        index_repository: IndexRepository,
-        fusion_service: FusionService,
-    ) -> None:
-        """Initialize the index query service."""
-        self.index_repository = index_repository
-        self.fusion_service = fusion_service
-
-    async def get_index_by_id(self, index_id: int) -> Index | None:
-        """Get an index by its ID."""
-        return await self.index_repository.get(index_id)
-
-    async def list_indexes(self) -> list[Index]:
-        """List all indexes."""
-        return await self.index_repository.all()
-
-    async def search_snippets(
-        self, request: MultiSearchRequest
-    ) -> list[SnippetWithContext]:
-        """Search snippets with filters.
-
-        Args:
-            request: The search request containing filters
-
-        Returns:
-            List of matching snippet items with context
-
-        """
-        return list(await self.index_repository.search(request))
-
-    async def perform_fusion(
-        self, rankings: list[list[FusionRequest]], k: float = 60
-    ) -> list[FusionResult]:
-        """Perform reciprocal rank fusion on search results."""
-        return self.fusion_service.reciprocal_rank_fusion(rankings, k)
-
-    async def get_snippets_by_ids(self, ids: list[int]) -> list[SnippetWithContext]:
-        """Get snippets by their IDs."""
-        snippets = await self.index_repository.get_snippets_by_ids(ids)
-
-        # Return snippets in the same order as the ids
-        snippets.sort(key=lambda x: ids.index(x.snippet.id or 0))
-        return snippets

kodit/domain/services/index_service.py

@@ -1,267 +0,0 @@
-"""Pure domain service for Index aggregate operations."""
-
-from abc import ABC, abstractmethod
-from collections import defaultdict
-from pathlib import Path
-
-import structlog
-from pydantic import AnyUrl
-
-import kodit.domain.entities as domain_entities
-from kodit.application.factories.reporting_factory import create_noop_operation
-from kodit.application.services.reporting import ProgressTracker
-from kodit.domain.services.enrichment_service import EnrichmentDomainService
-from kodit.domain.value_objects import (
-    EnrichmentIndexRequest,
-    EnrichmentRequest,
-    FileProcessingStatus,
-    LanguageMapping,
-)
-from kodit.infrastructure.cloning.git.working_copy import GitWorkingCopyProvider
-from kodit.infrastructure.cloning.metadata import FileMetadataExtractor
-from kodit.infrastructure.git.git_utils import is_valid_clone_target
-from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
-from kodit.infrastructure.slicing.slicer import Slicer
-from kodit.utils.path_utils import path_from_uri
-
-
-class LanguageDetectionService(ABC):
-    """Abstract interface for language detection service."""
-
-    @abstractmethod
-    async def detect_language(self, file_path: Path) -> str:
-        """Detect the programming language of a file."""
-
-
-class IndexDomainService:
-    """Pure domain service for Index aggregate operations.
-
-    This service handles the full lifecycle of code indexing:
-    - Creating indexes for source repositories
-    - Cloning and processing source files
-    - Extracting and enriching code snippets
-    - Managing the complete Index aggregate
-    """
-
-    def __init__(
-        self,
-        language_detector: LanguageDetectionService,
-        enrichment_service: EnrichmentDomainService,
-        clone_dir: Path,
-    ) -> None:
-        """Initialize the index domain service."""
-        self._clone_dir = clone_dir
-        self._language_detector = language_detector
-        self._enrichment_service = enrichment_service
-        self.log = structlog.get_logger(__name__)
-
-    async def prepare_index(
-        self,
-        uri_or_path_like: str,  # Must include user/pass, etc
-        step: ProgressTracker | None = None,
-    ) -> domain_entities.WorkingCopy:
-        """Prepare an index by scanning files and creating working copy."""
-        step = step or create_noop_operation()
-        self.log.info("Preparing index")
-        sanitized_uri, source_type = self.sanitize_uri(uri_or_path_like)
-        self.log.info("Preparing source", uri=str(sanitized_uri))
-
-        if source_type == domain_entities.SourceType.FOLDER:
-            local_path = path_from_uri(str(sanitized_uri))
-        elif source_type == domain_entities.SourceType.GIT:
-            source_type = domain_entities.SourceType.GIT
-            git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
-            local_path = await git_working_copy_provider.prepare(uri_or_path_like, step)
-        else:
-            raise ValueError(f"Unsupported source: {uri_or_path_like}")
-
-        return domain_entities.WorkingCopy(
-            remote_uri=sanitized_uri,
-            cloned_path=local_path,
-            source_type=source_type,
-            files=[],
-        )
-
-    async def extract_snippets_from_index(
-        self,
-        index: domain_entities.Index,
-        step: ProgressTracker | None = None,
-    ) -> domain_entities.Index:
-        """Extract code snippets from files in the index."""
-        step = step or create_noop_operation()
-        file_count = len(index.source.working_copy.files)
-
-        self.log.info(
-            "Extracting snippets",
-            index_id=index.id,
-            file_count=file_count,
-        )
-
-        # Only create snippets for files that have been added or modified
-        files = index.source.working_copy.changed_files()
-        index.delete_snippets_for_files(files)
-
-        # Filter out deleted files - they don't exist on disk anymore
-        files = [
-            f for f in files if f.file_processing_status != FileProcessingStatus.DELETED
-        ]
-
-        # Create a set of languages to extract snippets for
-        extensions = {file.extension() for file in files}
-        lang_files_map: dict[str, list[domain_entities.File]] = defaultdict(list)
-        for ext in extensions:
-            try:
-                lang = LanguageMapping.get_language_for_extension(ext)
-                lang_files_map[lang].extend(
-                    file for file in files if file.extension() == ext
-                )
-            except ValueError as e:
-                self.log.debug("Skipping", error=str(e))
-                continue
-
-        self.log.info(
-            "Languages to process",
-            languages=lang_files_map.keys(),
-        )
-
-        # Calculate snippets for each language
-        slicer = Slicer()
-        step.set_total(len(lang_files_map.keys()))
-        for i, (lang, lang_files) in enumerate(lang_files_map.items()):
-            step.set_current(i)
-            s = slicer.extract_snippets(lang_files, language=lang)
-            index.snippets.extend(s)
-
-        return index
-
-    async def enrich_snippets_in_index(
-        self,
-        snippets: list[domain_entities.Snippet],
-        reporting_step: ProgressTracker | None = None,
-    ) -> list[domain_entities.Snippet]:
-        """Enrich snippets with AI-generated summaries."""
-        reporting_step = reporting_step or create_noop_operation()
-        if not snippets or len(snippets) == 0:
-            reporting_step.skip("No snippets to enrich")
-            return snippets
-
-        reporting_step.set_total(len(snippets))
-        snippet_map = {snippet.id: snippet for snippet in snippets if snippet.id}
-
-        enrichment_request = EnrichmentIndexRequest(
-            requests=[
-                EnrichmentRequest(snippet_id=snippet_id, text=snippet.original_text())
-                for snippet_id, snippet in snippet_map.items()
-            ]
-        )
-
-        processed = 0
-        async for result in self._enrichment_service.enrich_documents(
-            enrichment_request
-        ):
-            snippet_map[result.snippet_id].add_summary(result.text)
-
-            processed += 1
-            reporting_step.set_current(processed)
-
-        return list(snippet_map.values())
-
-    def sanitize_uri(
-        self, uri_or_path_like: str
-    ) -> tuple[AnyUrl, domain_entities.SourceType]:
-        """Convert a URI or path-like string to a URI."""
-        # First, check if it's a local directory (more reliable than git check)
-        if Path(uri_or_path_like).is_dir():
-            return (
-                domain_entities.WorkingCopy.sanitize_local_path(uri_or_path_like),
-                domain_entities.SourceType.FOLDER,
-            )
-
-        # Then check if it's git-clonable
-        if is_valid_clone_target(uri_or_path_like):
-            return (
-                domain_entities.WorkingCopy.sanitize_git_url(uri_or_path_like),
-                domain_entities.SourceType.GIT,
-            )
-
-        raise ValueError(f"Unsupported source: {uri_or_path_like}")
-
-    async def refresh_working_copy(
-        self,
-        working_copy: domain_entities.WorkingCopy,
-        step: ProgressTracker | None = None,
-    ) -> domain_entities.WorkingCopy:
-        """Refresh the working copy."""
-        step = step or create_noop_operation()
-        metadata_extractor = FileMetadataExtractor(working_copy.source_type)
-        if working_copy.source_type == domain_entities.SourceType.GIT:
-            git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
-            await git_working_copy_provider.sync(str(working_copy.remote_uri), step)
-
-        current_file_paths = working_copy.list_filesystem_paths(
-            GitIgnorePatternProvider(working_copy.cloned_path)
-        )
-
-        previous_files_map = {file.as_path(): file for file in working_copy.files}
-
-        # Calculate different sets of files
-        deleted_file_paths = set(previous_files_map.keys()) - set(current_file_paths)
-        new_file_paths = set(current_file_paths) - set(previous_files_map.keys())
-        modified_file_paths = set(current_file_paths) & set(previous_files_map.keys())
-        num_files_to_process = (
-            len(deleted_file_paths) + len(new_file_paths) + len(modified_file_paths)
-        )
-        self.log.info(
-            "Refreshing working copy",
-            num_deleted=len(deleted_file_paths),
-            num_new=len(new_file_paths),
-            num_modified=len(modified_file_paths),
-            num_total_changes=num_files_to_process,
-            num_dirty=len(working_copy.dirty_files()),
-        )
-
-        # Setup reporter
-        processed = 0
-        step.set_total(num_files_to_process)
-
-        # First check to see if any files have been deleted
-        for file_path in deleted_file_paths:
-            processed += 1
-            step.set_current(processed)
-            previous_files_map[
-                file_path
-            ].file_processing_status = domain_entities.FileProcessingStatus.DELETED
-
-        # Then check to see if there are any new files
-        for file_path in new_file_paths:
-            processed += 1
-            step.set_current(processed)
-            try:
-                working_copy.files.append(
-                    await metadata_extractor.extract(file_path=file_path)
-                )
-            except (OSError, ValueError) as e:
-                self.log.debug("Skipping file", file=str(file_path), error=str(e))
-                continue
-
-        # Finally check if there are any modified files
-        for file_path in modified_file_paths:
-            processed += 1
-            step.set_current(processed)
-            try:
-                previous_file = previous_files_map[file_path]
-                new_file = await metadata_extractor.extract(file_path=file_path)
-                if previous_file.sha256 != new_file.sha256:
-                    previous_file.file_processing_status = (
-                        domain_entities.FileProcessingStatus.MODIFIED
-                    )
-            except (OSError, ValueError) as e:
-                self.log.info("Skipping file", file=str(file_path), error=str(e))
-                continue
-
-        return working_copy
-
-    async def delete_index(self, index: domain_entities.Index) -> None:
-        """Delete an index."""
-        # Delete the working copy
-        index.source.working_copy.delete()