kodit 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

kodit/domain/services/index_service.py

@@ -0,0 +1,323 @@
+"""Pure domain service for Index aggregate operations."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from pathlib import Path
+
+import structlog
+from pydantic import AnyUrl
+
+import kodit.domain.entities as domain_entities
+from kodit.domain.interfaces import ProgressCallback
+from kodit.domain.services.enrichment_service import EnrichmentDomainService
+from kodit.domain.value_objects import (
+    EnrichmentIndexRequest,
+    EnrichmentRequest,
+    SnippetExtractionRequest,
+    SnippetExtractionResult,
+    SnippetExtractionStrategy,
+)
+from kodit.infrastructure.cloning.git.working_copy import GitWorkingCopyProvider
+from kodit.infrastructure.cloning.metadata import FileMetadataExtractor
+from kodit.infrastructure.git.git_utils import is_valid_clone_target
+from kodit.infrastructure.ignore.ignore_pattern_provider import GitIgnorePatternProvider
+from kodit.reporting import Reporter
+from kodit.utils.path_utils import path_from_uri
+
+
+class LanguageDetectionService(ABC):
+    """Abstract interface for language detection service."""
+
+    @abstractmethod
+    async def detect_language(self, file_path: Path) -> str:
+        """Detect the programming language of a file."""
+
+
+class SnippetExtractor(ABC):
+    """Abstract interface for snippet extraction."""
+
+    @abstractmethod
+    async def extract(self, file_path: Path, language: str) -> list[str]:
+        """Extract snippets from a file."""
+
+
+class IndexDomainService:
+    """Pure domain service for Index aggregate operations.
+
+    This service handles the full lifecycle of code indexing:
+    - Creating indexes for source repositories
+    - Cloning and processing source files
+    - Extracting and enriching code snippets
+    - Managing the complete Index aggregate
+    """
+
+    def __init__(
+        self,
+        language_detector: LanguageDetectionService,
+        snippet_extractors: Mapping[SnippetExtractionStrategy, SnippetExtractor],
+        enrichment_service: EnrichmentDomainService,
+        clone_dir: Path,
+    ) -> None:
+        """Initialize the index domain service."""
+        self._clone_dir = clone_dir
+        self._language_detector = language_detector
+        self._snippet_extractors = snippet_extractors
+        self._enrichment_service = enrichment_service
+        self.log = structlog.get_logger(__name__)
+
+    async def prepare_index(
+        self,
+        uri_or_path_like: str,  # Must include user/pass, etc
+        progress_callback: ProgressCallback | None = None,
+    ) -> domain_entities.WorkingCopy:
+        """Prepare an index by scanning files and creating working copy."""
+        sanitized_uri, source_type = self.sanitize_uri(uri_or_path_like)
+        reporter = Reporter(self.log, progress_callback)
+        self.log.info("Preparing source", uri=str(sanitized_uri))
+
+        if source_type == domain_entities.SourceType.FOLDER:
+            await reporter.start("prepare_index", 1, "Scanning source...")
+            local_path = path_from_uri(str(sanitized_uri))
+        elif source_type == domain_entities.SourceType.GIT:
+            source_type = domain_entities.SourceType.GIT
+            git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
+            await reporter.start("prepare_index", 1, "Cloning source...")
+            local_path = await git_working_copy_provider.prepare(uri_or_path_like)
+            await reporter.done("prepare_index")
+        else:
+            raise ValueError(f"Unsupported source: {uri_or_path_like}")
+
+        await reporter.done("prepare_index")
+
+        return domain_entities.WorkingCopy(
+            remote_uri=sanitized_uri,
+            cloned_path=local_path,
+            source_type=source_type,
+            files=[],
+        )
+
+    async def extract_snippets_from_index(
+        self,
+        index: domain_entities.Index,
+        strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED,
+        progress_callback: ProgressCallback | None = None,
+    ) -> domain_entities.Index:
+        """Extract code snippets from files in the index."""
+        file_count = len(index.source.working_copy.files)
+
+        self.log.info(
+            "Extracting snippets",
+            index_id=index.id,
+            file_count=file_count,
+            strategy=strategy.value,
+        )
+
+        # Only create snippets for files that have been added or modified
+        files = index.source.working_copy.changed_files()
+        index.delete_snippets_for_files(files)
+
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start(
+            "extract_snippets", len(files), "Extracting code snippets..."
+        )
+
+        new_snippets = []
+        for i, domain_file in enumerate(files, 1):
+            try:
+                # Extract snippets from file
+                request = SnippetExtractionRequest(
+                    file_path=domain_file.as_path(), strategy=strategy
+                )
+                result = await self._extract_snippets(request)
+                for snippet_text in result.snippets:
+                    snippet = domain_entities.Snippet(
+                        derives_from=[domain_file],
+                    )
+                    snippet.add_original_content(snippet_text, result.language)
+                    new_snippets.append(snippet)
+
+            except (OSError, ValueError) as e:
+                self.log.debug(
+                    "Skipping file for snippet extraction",
+                    file_uri=str(domain_file.uri),
+                    error=str(e),
+                )
+                continue
+
+            await reporter.step(
+                "extract_snippets", i, len(files), f"Processed {domain_file.uri.path}"
+            )
+
+        index.snippets.extend(new_snippets)
+        await reporter.done("extract_snippets")
+        return index
+
+    async def enrich_snippets_in_index(
+        self,
+        snippets: list[domain_entities.Snippet],
+        progress_callback: ProgressCallback | None = None,
+    ) -> list[domain_entities.Snippet]:
+        """Enrich snippets with AI-generated summaries."""
+        if not snippets or len(snippets) == 0:
+            return snippets
+
+        reporter = Reporter(self.log, progress_callback)
+        await reporter.start("enrichment", len(snippets), "Enriching snippets...")
+
+        snippet_map = {snippet.id: snippet for snippet in snippets if snippet.id}
+
+        enrichment_request = EnrichmentIndexRequest(
+            requests=[
+                EnrichmentRequest(snippet_id=snippet_id, text=snippet.original_text())
+                for snippet_id, snippet in snippet_map.items()
+            ]
+        )
+
+        processed = 0
+        async for result in self._enrichment_service.enrich_documents(
+            enrichment_request
+        ):
+            snippet_map[result.snippet_id].add_summary(result.text)
+
+            processed += 1
+            await reporter.step(
+                "enrichment", processed, len(snippets), "Enriching snippets..."
+            )
+
+        await reporter.done("enrichment")
+        return list(snippet_map.values())
+
+    async def _extract_snippets(
+        self, request: SnippetExtractionRequest
+    ) -> SnippetExtractionResult:
+        # Domain logic: validate file exists
+        if not request.file_path.exists():
+            raise ValueError(f"File does not exist: {request.file_path}")
+
+        # Domain logic: detect language
+        language = await self._language_detector.detect_language(request.file_path)
+
+        # Domain logic: choose strategy and extractor
+        if request.strategy not in self._snippet_extractors:
+            raise ValueError(f"Unsupported extraction strategy: {request.strategy}")
+
+        extractor = self._snippet_extractors[request.strategy]
+        snippets = await extractor.extract(request.file_path, language)
+
+        # Domain logic: filter out empty snippets
+        filtered_snippets = [snippet for snippet in snippets if snippet.strip()]
+
+        return SnippetExtractionResult(snippets=filtered_snippets, language=language)
+
+    def sanitize_uri(
+        self, uri_or_path_like: str
+    ) -> tuple[AnyUrl, domain_entities.SourceType]:
+        """Convert a URI or path-like string to a URI."""
+        # First, check if it's a local directory (more reliable than git check)
+        if Path(uri_or_path_like).is_dir():
+            return (
+                domain_entities.WorkingCopy.sanitize_local_path(uri_or_path_like),
+                domain_entities.SourceType.FOLDER,
+            )
+
+        # Then check if it's git-clonable
+        if is_valid_clone_target(uri_or_path_like):
+            return (
+                domain_entities.WorkingCopy.sanitize_git_url(uri_or_path_like),
+                domain_entities.SourceType.GIT,
+            )
+
+        raise ValueError(f"Unsupported source: {uri_or_path_like}")
+
+    async def refresh_working_copy(
+        self,
+        working_copy: domain_entities.WorkingCopy,
+        progress_callback: ProgressCallback | None = None,
+    ) -> domain_entities.WorkingCopy:
+        """Refresh the working copy."""
+        metadata_extractor = FileMetadataExtractor(working_copy.source_type)
+        reporter = Reporter(self.log, progress_callback)
+
+        if working_copy.source_type == domain_entities.SourceType.GIT:
+            git_working_copy_provider = GitWorkingCopyProvider(self._clone_dir)
+            await git_working_copy_provider.sync(str(working_copy.remote_uri))
+
+        current_file_paths = working_copy.list_filesystem_paths(
+            GitIgnorePatternProvider(working_copy.cloned_path)
+        )
+
+        previous_files_map = {file.as_path(): file for file in working_copy.files}
+
+        # Calculate different sets of files
+        deleted_file_paths = set(previous_files_map.keys()) - set(current_file_paths)
+        new_file_paths = set(current_file_paths) - set(previous_files_map.keys())
+        modified_file_paths = set(current_file_paths) & set(previous_files_map.keys())
+        num_files_to_process = (
+            len(deleted_file_paths) + len(new_file_paths) + len(modified_file_paths)
+        )
+        self.log.info(
+            "Refreshing working copy",
+            num_deleted=len(deleted_file_paths),
+            num_new=len(new_file_paths),
+            num_modified=len(modified_file_paths),
+            num_total_changes=num_files_to_process,
+            num_dirty=len(working_copy.dirty_files()),
+        )
+
+        # Setup reporter
+        processed = 0
+        await reporter.start(
+            "refresh_working_copy", num_files_to_process, "Refreshing working copy..."
+        )
+
+        # First check to see if any files have been deleted
+        for file_path in deleted_file_paths:
+            processed += 1
+            await reporter.step(
+                "refresh_working_copy",
+                processed,
+                num_files_to_process,
+                f"Deleted {file_path.name}",
+            )
+            previous_files_map[
+                file_path
+            ].file_processing_status = domain_entities.FileProcessingStatus.DELETED
+
+        # Then check to see if there are any new files
+        for file_path in new_file_paths:
+            processed += 1
+            await reporter.step(
+                "refresh_working_copy",
+                processed,
+                num_files_to_process,
+                f"New {file_path.name}",
+            )
+            try:
+                working_copy.files.append(
+                    await metadata_extractor.extract(file_path=file_path)
+                )
+            except (OSError, ValueError) as e:
+                self.log.info("Skipping file", file=str(file_path), error=str(e))
+                continue
+
+        # Finally check if there are any modified files
+        for file_path in modified_file_paths:
+            processed += 1
+            await reporter.step(
+                "refresh_working_copy",
+                processed,
+                num_files_to_process,
+                f"Modified {file_path.name}",
+            )
+            try:
+                previous_file = previous_files_map[file_path]
+                new_file = await metadata_extractor.extract(file_path=file_path)
+                if previous_file.sha256 != new_file.sha256:
+                    previous_file.file_processing_status = (
+                        domain_entities.FileProcessingStatus.MODIFIED
+                    )
+            except (OSError, ValueError) as e:
+                self.log.info("Skipping file", file=str(file_path), error=str(e))
+                continue
+
+        return working_copy

kodit/domain/value_objects.py

@@ -1,16 +1,129 @@
-"""Domain value objects and DTOs."""
+"""Pure domain value objects and DTOs."""
 
+import json
 from dataclasses import dataclass
 from datetime import datetime
-from enum import Enum
+from enum import Enum, IntEnum
 from pathlib import Path
-from typing import Any, ClassVar
+from typing import ClassVar
 
-from sqlalchemy import JSON, DateTime, Integer, Text
-from sqlalchemy.orm import Mapped, mapped_column
+from pydantic import BaseModel
 
-from kodit.domain.entities import Base
-from kodit.domain.enums import SnippetExtractionStrategy
+
+class SourceType(IntEnum):
+    """The type of source."""
+
+    UNKNOWN = 0
+    FOLDER = 1
+    GIT = 2
+
+
+class SnippetContentType(IntEnum):
+    """Type of snippet content."""
+
+    UNKNOWN = 0
+    ORIGINAL = 1
+    SUMMARY = 2
+
+
+class SnippetContent(BaseModel):
+    """Snippet content domain value object."""
+
+    type: SnippetContentType
+    value: str
+    language: str
+
+
+class SnippetSearchResult(BaseModel):
+    """Domain result object for snippet searches."""
+
+    snippet_id: int
+    content: str
+    summary: str
+    score: float
+    file_path: Path
+    language: str | None = None
+    authors: list[str] = []
+
+
+@dataclass(frozen=True)
+class LanguageExtensions:
+    """Value object for language to file extension mappings."""
+
+    language: str
+    extensions: list[str]
+
+    @classmethod
+    def get_supported_languages(cls) -> list[str]:
+        """Get all supported programming languages."""
+        return [
+            "python",
+            "javascript",
+            "typescript",
+            "java",
+            "c",
+            "cpp",
+            "csharp",
+            "go",
+            "rust",
+            "php",
+            "ruby",
+            "swift",
+            "kotlin",
+            "scala",
+            "r",
+            "sql",
+            "html",
+            "css",
+            "json",
+            "yaml",
+            "xml",
+            "markdown",
+            "shell",
+        ]
+
+    @classmethod
+    def get_extensions_for_language(cls, language: str) -> list[str]:
+        """Get file extensions for a given language."""
+        language_map = {
+            "python": [".py", ".pyw", ".pyi"],
+            "javascript": [".js", ".jsx", ".mjs"],
+            "typescript": [".ts", ".tsx"],
+            "java": [".java"],
+            "c": [".c", ".h"],
+            "cpp": [".cpp", ".cc", ".cxx", ".hpp", ".hxx"],
+            "csharp": [".cs"],
+            "go": [".go"],
+            "rust": [".rs"],
+            "php": [".php"],
+            "ruby": [".rb"],
+            "swift": [".swift"],
+            "kotlin": [".kt", ".kts"],
+            "scala": [".scala", ".sc"],
+            "r": [".r", ".R"],
+            "sql": [".sql"],
+            "html": [".html", ".htm"],
+            "css": [".css", ".scss", ".sass", ".less"],
+            "json": [".json"],
+            "yaml": [".yaml", ".yml"],
+            "xml": [".xml"],
+            "markdown": [".md", ".markdown"],
+            "shell": [".sh", ".bash", ".zsh", ".fish"],
+        }
+        return language_map.get(language.lower(), [])
+
+    @classmethod
+    def is_supported_language(cls, language: str) -> bool:
+        """Check if a language is supported."""
+        return language.lower() in cls.get_supported_languages()
+
+    @classmethod
+    def get_extensions_or_fallback(cls, language: str) -> list[str]:
+        """Get extensions for language or return language as extension if not found."""
+        language_lower = language.lower()
+        if cls.is_supported_language(language_lower):
+            return cls.get_extensions_for_language(language_lower)
+        return [language_lower]
 
 
 class SearchType(Enum):
@@ -21,14 +134,6 @@ class SearchType(Enum):
     HYBRID = "hybrid"
 
 
-@dataclass
-class SnippetExtractionRequest:
-    """Domain model for snippet extraction request."""
-
-    file_path: Path
-    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED
-
-
 @dataclass
 class SnippetExtractionResult:
     """Domain model for snippet extraction result."""
@@ -100,6 +205,7 @@ class SnippetSearchFilters:
     created_after: datetime | None = None
     created_before: datetime | None = None
     source_repo: str | None = None
+    file_path: str | None = None
 
     @classmethod
     def from_cli_params(
@@ -175,25 +281,90 @@ class MultiSearchRequest:
 
 @dataclass
 class MultiSearchResult:
-    """Domain model for multi-modal search result."""
+    """Enhanced search result with comprehensive snippet metadata."""
 
     id: int
-    uri: str
     content: str
     original_scores: list[float]
+    source_uri: str
+    relative_path: str
+    language: str
+    authors: list[str]
+    created_at: datetime
+    summary: str
 
     def __str__(self) -> str:
-        """Return formatted string representation for all snippet display."""
+        """Return enhanced formatted string representation."""
         lines = [
-            "-" * 80,
-            f"ID: {self.id} | {self.uri}",
-            f"Original scores: {self.original_scores}",
-            self.content,
-            "-" * 80,
-            "",
+            "---",
+            f"id: {self.id}",
+            f"source: {self.source_uri}",
+            f"path: {self.relative_path}",
+            f"lang: {self.language}",
+            f"created: {self.created_at.isoformat()}",
+            f"authors: {', '.join(self.authors)}",
+            f"scores: {self.original_scores}",
+            "---",
+            f"{self.summary}\n",
+            f"```{self.language}",
+            f"{self.content}",
+            "```\n",
         ]
         return "\n".join(lines)
 
+    def to_json(self) -> str:
+        """Return LLM-optimized JSON representation following the compact schema."""
+        json_obj = {
+            "id": self.id,
+            "source": self.source_uri,
+            "path": self.relative_path,
+            "lang": self.language.lower(),
+            "created": self.created_at.isoformat() if self.created_at else "",
+            "author": ", ".join(self.authors),
+            "score": self.original_scores,
+            "code": self.content,
+            "summary": self.summary,
+        }
+
+        return json.dumps(json_obj, separators=(",", ":"))
+
+    @classmethod
+    def to_jsonlines(cls, results: list["MultiSearchResult"]) -> str:
+        """Convert multiple MultiSearchResult objects to JSON Lines format.
+
+        Args:
+            results: List of MultiSearchResult objects
+            include_summary: Whether to include summary fields
+
+        Returns:
+            JSON Lines string (one JSON object per line)
+
+        """
+        return "\n".join(result.to_json() for result in results)
+
+    @classmethod
+    def to_string(cls, results: list["MultiSearchResult"]) -> str:
+        """Convert multiple MultiSearchResult objects to a string."""
+        return "\n\n".join(str(result) for result in results)
+
+    @staticmethod
+    def calculate_relative_path(file_path: str, source_path: str) -> str:
+        """Calculate relative path from source root."""
+        try:
+            return str(Path(file_path).relative_to(Path(source_path)))
+        except ValueError:
+            # If file_path is not relative to source_path, return the file name
+            return Path(file_path).name
+
+    @staticmethod
+    def detect_language_from_extension(extension: str) -> str:
+        """Detect programming language from file extension."""
+        try:
+            return LanguageMapping.get_language_for_extension(extension).title()
+        except ValueError:
+            # Unknown extension, return a default
+            return "Unknown"
+
 
 @dataclass
 class FusionRequest:
@@ -291,39 +462,6 @@ class IndexView:
     source: str | None = None
 
 
-@dataclass
-class SnippetListItem:
-    """Domain model for snippet list item with file information."""
-
-    id: int
-    file_path: str
-    content: str
-    source_uri: str
-
-
-@dataclass
-class FileInfo:
-    """Domain model for file information."""
-
-    uri: str
-
-
-@dataclass
-class SnippetInfo:
-    """Domain model for snippet information."""
-
-    id: int
-    content: str
-
-
-@dataclass
-class SnippetWithFile:
-    """Domain model for snippet with associated file information."""
-
-    file: FileInfo
-    snippet: SnippetInfo
-
-
 class LanguageMapping:
     """Value object for language-to-extension mappings.
 
@@ -493,38 +631,33 @@ class LanguageMapping:
         return [language_lower]
 
 
-# Database models for value objects
-class BM25DocumentModel(Base):
-    """BM25 document model."""
-
-    __tablename__ = "bm25_documents"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    content: Mapped[str] = mapped_column(Text, nullable=False)
-    document_metadata: Mapped[dict[str, Any] | None] = mapped_column(
-        JSON, nullable=True
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False
-    )
-
-
-class VectorDocumentModel(Base):
-    """Vector document model."""
-
-    __tablename__ = "vector_documents"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    content: Mapped[str] = mapped_column(Text, nullable=False)
-    document_metadata: Mapped[dict[str, Any] | None] = mapped_column(
-        JSON, nullable=True
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False
-    )
+class SnippetQuery(BaseModel):
+    """Domain query object for snippet searches."""
+
+    text: str
+    search_type: SearchType = SearchType.HYBRID
+    filters: SnippetSearchFilters = SnippetSearchFilters()
+    top_k: int = 10
+
+
+class SnippetExtractionStrategy(str, Enum):
+    """Different strategies for extracting snippets from files."""
+
+    METHOD_BASED = "method_based"
+
+
+@dataclass
+class SnippetExtractionRequest:
+    """Domain model for snippet extraction request."""
+
+    file_path: Path
+    strategy: SnippetExtractionStrategy = SnippetExtractionStrategy.METHOD_BASED
+
+
+class FileProcessingStatus(IntEnum):
+    """File processing status."""
+
+    CLEAN = 0
+    ADDED = 1
+    MODIFIED = 2
+    DELETED = 3