kodit 0.4.3__py3-none-any.whl → 0.5.0__py3-none-any.whl

kodit/cli_utils.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any
 
 import click
 
-from kodit.infrastructure.api.client import IndexClient, SearchClient
+from kodit.infrastructure.api.client import SearchClient
 
 if TYPE_CHECKING:
     from kodit.config import AppContext
@@ -37,7 +37,7 @@ def with_client(f: Callable) -> Callable:
             inner_func = getattr(
                 getattr(session_wrapped, "__wrapped__", session_wrapped),
                 "__wrapped__",
-                session_wrapped
+                session_wrapped,
             )
 
             # Get database session manually
@@ -47,13 +47,6 @@ def with_client(f: Callable) -> Callable:
         else:
             # Remote mode - use API clients
             clients = {
-                "index_client": IndexClient(
-                    base_url=app_context.remote.server_url or "",
-                    api_key=app_context.remote.api_key,
-                    timeout=app_context.remote.timeout,
-                    max_retries=app_context.remote.max_retries,
-                    verify_ssl=app_context.remote.verify_ssl,
-                ),
                 "search_client": SearchClient(
                     base_url=app_context.remote.server_url or "",
                     api_key=app_context.remote.api_key,

kodit/config.py

@@ -14,9 +14,7 @@ import structlog
 from pydantic import BaseModel, Field, field_validator
 from pydantic_settings import (
     BaseSettings,
-    EnvSettingsSource,
     NoDecode,
-    PydanticBaseSettingsSource,
     SettingsConfigDict,
 )
 
@@ -91,46 +89,12 @@ class Search(BaseModel):
     provider: Literal["sqlite", "vectorchord"] = Field(default="sqlite")
 
 
-class AutoIndexingSource(BaseModel):
-    """Configuration for a single auto-indexing source."""
-
-    uri: str = Field(description="URI of the source to index (git URL or local path)")
-
-
-class AutoIndexingConfig(BaseModel):
-    """Configuration for auto-indexing."""
-
-    sources: list[AutoIndexingSource] = Field(
-        default_factory=list, description="List of sources to auto-index"
-    )
-
-    @field_validator("sources", mode="before")
-    @classmethod
-    def parse_sources(cls, v: Any) -> Any:
-        """Parse sources from environment variables or other formats."""
-        if v is None:
-            return []
-        if isinstance(v, list):
-            return v
-        if isinstance(v, dict):
-            # Handle case where env vars are numbered keys like {'0': {'uri': '...'}}
-            sources = []
-            i = 0
-            while str(i) in v:
-                source_data = v[str(i)]
-                if isinstance(source_data, dict) and "uri" in source_data:
-                    sources.append(AutoIndexingSource(uri=source_data["uri"]))
-                i += 1
-            return sources
-        return v
-
-
 class PeriodicSyncConfig(BaseModel):
     """Configuration for periodic/scheduled syncing."""
 
     enabled: bool = Field(default=True, description="Enable periodic sync")
     interval_seconds: float = Field(
-        default=1800, description="Interval between automatic syncs in seconds"
+        default=1800, description="Interval between periodic syncs in seconds"
     )
     retry_attempts: int = Field(
         default=3, description="Number of retry attempts for failed syncs"
@@ -147,36 +111,6 @@ class RemoteConfig(BaseModel):
     verify_ssl: bool = Field(default=True, description="Verify SSL certificates")
 
 
-class CustomAutoIndexingEnvSource(EnvSettingsSource):
-    """Custom environment source for parsing AutoIndexingConfig."""
-
-    def __call__(self) -> dict[str, Any]:
-        """Load settings from env vars with custom auto-indexing parsing."""
-        d: dict[str, Any] = {}
-
-        # First get the standard env vars
-        env_vars = super().__call__()
-        d.update(env_vars)
-
-        # Custom parsing for auto-indexing sources
-        auto_indexing_sources = []
-        i = 0
-        while True:
-            # Note: env_vars keys are lowercase due to Pydantic Settings normalization
-            uri_key = f"auto_indexing_sources_{i}_uri"
-            if uri_key in self.env_vars:
-                uri_value = self.env_vars[uri_key]
-                auto_indexing_sources.append({"uri": uri_value})
-                i += 1
-            else:
-                break
-
-        if auto_indexing_sources:
-            d["auto_indexing"] = {"sources": auto_indexing_sources}
-
-        return d
-
-
 class AppContext(BaseSettings):
     """Global context for the kodit project. Provides a shared state for the app."""
 
@@ -189,30 +123,6 @@ class AppContext(BaseSettings):
         extra="ignore",
     )
 
-    @classmethod
-    def settings_customise_sources(
-        cls,
-        settings_cls: type[BaseSettings],
-        init_settings: PydanticBaseSettingsSource,
-        env_settings: PydanticBaseSettingsSource,  # noqa: ARG003
-        dotenv_settings: PydanticBaseSettingsSource,
-        file_secret_settings: PydanticBaseSettingsSource,
-    ) -> tuple[PydanticBaseSettingsSource, ...]:
-        """Customize settings sources to use custom auto-indexing parsing."""
-        custom_env_settings = CustomAutoIndexingEnvSource(
-            settings_cls,
-            env_nested_delimiter=settings_cls.model_config.get("env_nested_delimiter"),
-            env_ignore_empty=settings_cls.model_config.get("env_ignore_empty", False),
-            env_parse_none_str=settings_cls.model_config.get("env_parse_none_str", ""),
-            env_parse_enums=settings_cls.model_config.get("env_parse_enums", None),
-        )
-        return (
-            init_settings,
-            custom_env_settings,
-            dotenv_settings,
-            file_secret_settings,
-        )
-
     data_dir: Path = Field(default=DEFAULT_BASE_DIR)
     db_url: str = Field(
         default_factory=lambda data: f"sqlite+aiosqlite:///{data['data_dir']}/kodit.db"
@@ -231,9 +141,6 @@ class AppContext(BaseSettings):
     default_search: Search = Field(
         default=Search(),
     )
-    auto_indexing: AutoIndexingConfig | None = Field(
-        default=AutoIndexingConfig(), description="Auto-indexing configuration"
-    )
     periodic_sync: PeriodicSyncConfig = Field(
         default=PeriodicSyncConfig(), description="Periodic sync configuration"
     )

kodit/database.py

@@ -2,10 +2,12 @@
 
 from collections.abc import Callable
 from pathlib import Path
+from typing import Any
 
 import structlog
 from alembic import command
 from alembic.config import Config as AlembicConfig
+from sqlalchemy import event
 from sqlalchemy.ext.asyncio import (
     AsyncSession,
     async_sessionmaker,
@@ -21,7 +23,42 @@ class Database:
     def __init__(self, db_url: str) -> None:
         """Initialize the database."""
         self.log = structlog.get_logger(__name__)
-        self.db_engine = create_async_engine(db_url, echo=False)
+
+        # Configure SQLite-specific connection arguments to prevent locking issues
+        connect_args = {}
+        if "sqlite" in db_url.lower():
+            connect_args = {
+                "timeout": 20,  # 20 second timeout for database operations
+                "check_same_thread": False,  # Allow use from different threads
+            }
+
+        self.db_engine = create_async_engine(
+            db_url,
+            echo=False,
+            connect_args=connect_args,
+        )
+
+        # Configure SQLite pragmas for better concurrency and performance
+        if "sqlite" in db_url.lower():
+
+            @event.listens_for(self.db_engine.sync_engine, "connect")
+            def set_sqlite_pragma(
+                dbapi_connection: Any, connection_record: Any
+            ) -> None:
+                del (
+                    connection_record
+                )  # Unused but required by SQLAlchemy event interface
+                cursor = dbapi_connection.cursor()
+                # Enable WAL mode for better concurrency
+                cursor.execute("PRAGMA journal_mode=WAL")
+                # Set busy timeout to prevent immediate locking failures
+                cursor.execute("PRAGMA busy_timeout=20000")
+                # Enable foreign key constraints
+                cursor.execute("PRAGMA foreign_keys=ON")
+                # Optimize for speed over safety (acceptable for indexing workloads)
+                cursor.execute("PRAGMA synchronous=NORMAL")
+                cursor.close()
+
         self.db_session_factory = async_sessionmaker(
             self.db_engine,
             class_=AsyncSession,

kodit/domain/{entities.py → entities/__init__.py}

@@ -1,7 +1,5 @@
 """Pure domain entities using Pydantic."""
 
-import shutil
-from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any, Protocol
@@ -10,17 +8,10 @@ from urllib.parse import urlparse, urlunparse
 from pydantic import AnyUrl, BaseModel
 
 from kodit.domain.value_objects import (
-    FileProcessingStatus,
-    QueuePriority,
     ReportingState,
-    SnippetContent,
-    SnippetContentType,
-    SourceType,
     TaskOperation,
-    TaskType,
     TrackableType,
 )
-from kodit.utils.path_utils import path_from_uri
 
 
 class IgnorePatternProvider(Protocol):
@@ -39,37 +30,9 @@ class Author(BaseModel):
     email: str
 
 
-class File(BaseModel):
-    """File domain entity."""
-
-    id: int | None = None  # Is populated by repository
-    created_at: datetime | None = None  # Is populated by repository
-    updated_at: datetime | None = None  # Is populated by repository
-    uri: AnyUrl
-    sha256: str
-    authors: list[Author]
-    mime_type: str
-    file_processing_status: FileProcessingStatus
-
-    def as_path(self) -> Path:
-        """Return the file as a path."""
-        return path_from_uri(str(self.uri))
-
-    def extension(self) -> str:
-        """Return the file extension."""
-        return Path(self.as_path()).suffix.lstrip(".")
-
-
 class WorkingCopy(BaseModel):
     """Working copy value object representing cloned source location."""
 
-    created_at: datetime | None = None  # Is populated by repository
-    updated_at: datetime | None = None  # Is populated by repository
-    remote_uri: AnyUrl
-    cloned_path: Path
-    source_type: SourceType
-    files: list[File]
-
     @classmethod
     def sanitize_local_path(cls, path: str) -> AnyUrl:
         """Sanitize a local path."""
@@ -100,26 +63,54 @@ class WorkingCopy(BaseModel):
             "ssh://git@github.com/user/repo.git"
 
         """
-        # Handle SSH URLs (they don't have credentials in the URL format)
+        # Handle different URL types
+        if not url:
+            raise ValueError("URL is required")
+
         if url.startswith("git@"):
-            # Convert git@host:path to ssh://git@host/path format for AnyUrl
-            # This maintains the same semantic meaning while making it a valid URL
-            if ":" in url and not url.startswith("ssh://"):
-                host_path = url[4:]  # Remove "git@"
-                if ":" in host_path:
-                    host, path = host_path.split(":", 1)
-                    ssh_url = f"ssh://git@{host}/{path}"
-                    return AnyUrl(ssh_url)
-            return AnyUrl(url)
+            return cls._handle_ssh_url(url)
         if url.startswith("ssh://"):
             return AnyUrl(url)
-
-        # Handle file URLs
         if url.startswith("file://"):
             return AnyUrl(url)
 
+        # Try local path conversion
+        local_url = cls._try_local_path_conversion(url)
+        if local_url:
+            return local_url
+
+        # Handle HTTPS URLs with credentials
+        return cls._sanitize_https_url(url)
+
+    @classmethod
+    def _handle_ssh_url(cls, url: str) -> AnyUrl:
+        """Handle SSH URL conversion."""
+        if ":" in url and not url.startswith("ssh://"):
+            host_path = url[4:]  # Remove "git@"
+            if ":" in host_path:
+                host, path = host_path.split(":", 1)
+                return AnyUrl(f"ssh://git@{host}/{path}")
+        return AnyUrl(url)
+
+    @classmethod
+    def _try_local_path_conversion(cls, url: str) -> AnyUrl | None:
+        """Try to convert local paths to file:// URLs."""
+        from pathlib import Path
+
+        try:
+            path = Path(url)
+            if path.exists() or url.startswith(("/", "./", "../")) or url == ".":
+                absolute_path = path.resolve()
+                return AnyUrl(f"file://{absolute_path}")
+        except OSError:
+            # Path operations failed, not a local path
+            pass
+        return None
+
+    @classmethod
+    def _sanitize_https_url(cls, url: str) -> AnyUrl:
+        """Remove credentials from HTTPS URLs."""
         try:
-            # Parse the URL
             parsed = urlparse(url)
 
             # If there are no credentials, return the URL as-is
@@ -127,7 +118,6 @@ class WorkingCopy(BaseModel):
                 return AnyUrl(url)
 
             # Reconstruct the URL without credentials
-            # scheme, netloc (without username/password), path, params, query, fragment
             sanitized_netloc = parsed.hostname
             if parsed.port:
                 sanitized_netloc = f"{parsed.hostname}:{parsed.port}"
@@ -144,65 +134,9 @@ class WorkingCopy(BaseModel):
                     )
                 )
             )
-
         except Exception as e:
             raise ValueError(f"Invalid URL: {url}") from e
 
-    def modified_or_deleted_files(self) -> list[File]:
-        """Return the modified or deleted files."""
-        return [
-            file
-            for file in self.files
-            if file.file_processing_status
-            in (FileProcessingStatus.MODIFIED, FileProcessingStatus.DELETED)
-        ]
-
-    def list_filesystem_paths(
-        self, ignore_provider: IgnorePatternProvider
-    ) -> list[Path]:
-        """List the filesystem paths of the files in the working copy."""
-        if not self.cloned_path.exists():
-            raise ValueError(f"Cloned path does not exist: {self.cloned_path}")
-
-        return [
-            f
-            for f in self.cloned_path.rglob("*")
-            if f.is_file() and not ignore_provider.should_ignore(f)
-        ]
-
-    def dirty_files(self) -> list[File]:
-        """Return the dirty files."""
-        return [
-            file
-            for file in self.files
-            if file.file_processing_status
-            in (FileProcessingStatus.MODIFIED, FileProcessingStatus.ADDED)
-        ]
-
-    def changed_files(self) -> list[File]:
-        """Return the changed files."""
-        return [
-            file
-            for file in self.files
-            if file.file_processing_status != FileProcessingStatus.CLEAN
-        ]
-
-    def clear_file_processing_statuses(self) -> None:
-        """Clear the file processing statuses."""
-        # First remove any files that are marked for deletion
-        self.files = [
-            file
-            for file in self.files
-            if file.file_processing_status != FileProcessingStatus.DELETED
-        ]
-        # Then clear the statuses for the remaining files
-        for file in self.files:
-            file.file_processing_status = FileProcessingStatus.CLEAN
-
-    def delete(self) -> None:
-        """Delete the working copy."""
-        shutil.rmtree(self.cloned_path)
-
 
 class Source(BaseModel):
     """Source domain entity."""
@@ -213,74 +147,6 @@ class Source(BaseModel):
     working_copy: WorkingCopy
 
 
-class Snippet(BaseModel):
-    """Snippet domain entity."""
-
-    id: int | None = None  # Is populated by repository
-    created_at: datetime | None = None  # Is populated by repository
-    updated_at: datetime | None = None  # Is populated by repository
-    derives_from: list[File]
-    original_content: SnippetContent | None = None
-    summary_content: SnippetContent | None = None
-
-    def original_text(self) -> str:
-        """Return the original content of the snippet."""
-        if self.original_content is None:
-            return ""
-        return self.original_content.value
-
-    def summary_text(self) -> str:
-        """Return the summary content of the snippet."""
-        if self.summary_content is None:
-            return ""
-        return self.summary_content.value
-
-    def add_original_content(self, content: str, language: str) -> None:
-        """Add an original content to the snippet."""
-        self.original_content = SnippetContent(
-            type=SnippetContentType.ORIGINAL,
-            value=content,
-            language=language,
-        )
-
-    def add_summary(self, summary: str) -> None:
-        """Add a summary to the snippet."""
-        self.summary_content = SnippetContent(
-            type=SnippetContentType.SUMMARY,
-            value=summary,
-            language="markdown",
-        )
-
-
-class Index(BaseModel):
-    """Index domain entity."""
-
-    id: int
-    created_at: datetime
-    updated_at: datetime
-    source: Source
-    snippets: list[Snippet]
-
-    def delete_snippets_for_files(self, files: list[File]) -> None:
-        """Delete the snippets that derive from a list of files."""
-        self.snippets = [
-            snippet
-            for snippet in self.snippets
-            if not any(file in snippet.derives_from for file in files)
-        ]
-
-
-# FUTURE: Remove this type, use the domain to get the required information.
-@dataclass(frozen=True)
-class SnippetWithContext:
-    """Domain model for snippet with associated context information."""
-
-    source: Source
-    file: File
-    authors: list[Author]
-    snippet: Snippet
-
-
 class Task(BaseModel):
     """Represents an item in the queue waiting to be processed.
 
@@ -289,7 +155,7 @@ class Task(BaseModel):
     """
 
     id: str  # Is a unique key to deduplicate items in the queue
-    type: TaskType  # Task type
+    type: TaskOperation  # Task operation
     priority: int  # Priority (higher number = higher priority)
     payload: dict[str, Any]  # Task-specific data
 
@@ -297,33 +163,22 @@ class Task(BaseModel):
     updated_at: datetime | None = None  # Is populated by repository
 
     @staticmethod
-    def create(task_type: TaskType, priority: int, payload: dict[str, Any]) -> "Task":
+    def create(
+        operation: TaskOperation, priority: int, payload: dict[str, Any]
+    ) -> "Task":
         """Create a task."""
         return Task(
-            id=Task._create_id(task_type, payload),
-            type=task_type,
+            id=Task.create_id(operation, payload),
+            type=operation,
             priority=priority,
             payload=payload,
         )
 
     @staticmethod
-    def _create_id(task_type: TaskType, payload: dict[str, Any]) -> str:
+    def create_id(operation: TaskOperation, payload: dict[str, Any]) -> str:
         """Create a unique id for a task."""
-        if task_type == TaskType.INDEX_UPDATE:
-            return str(payload["index_id"])
-
-        raise ValueError(f"Unknown task type: {task_type}")
-
-    @staticmethod
-    def create_index_update_task(
-        index_id: int, priority: QueuePriority = QueuePriority.USER_INITIATED
-    ) -> "Task":
-        """Create an index update task."""
-        return Task.create(
-            task_type=TaskType.INDEX_UPDATE,
-            priority=priority.value,
-            payload={"index_id": index_id},
-        )
+        first_id = next(iter(payload.values()), None)
+        return f"{operation}:{first_id}"
 
 
 class TaskStatus(BaseModel):