docsif 0.1.0__py3-none-any.whl

sif/core/document.py

@@ -0,0 +1,125 @@
+"""Document domain entity."""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Protocol
+
+
+@dataclass
+class DocumentMetadata:
+    """Metadata extracted from document frontmatter."""
+
+    title: str | None = None
+    author: str | None = None
+    date: datetime | None = None
+    tags: list[str] = field(default_factory=list)
+    categories: list[str] = field(default_factory=list)
+    custom: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class DocumentChunk:
+    """A chunk of a document with embedding information."""
+
+    id: str
+    document_id: str
+    content: str
+    start_line: int
+    end_line: int
+    token_count: int
+    embedding_id: str | None = None
+    embedding: list[float] | None = None
+
+    def __post_init__(self) -> None:
+        """Validate chunk data."""
+        if self.start_line > self.end_line:
+            raise ValueError("start_line cannot be greater than end_line")
+
+
+@dataclass
+class Document:
+    """Domain entity representing a document.
+
+    A document is a single markdown file that has been indexed.
+    It contains the document content, metadata, and chunks.
+    """
+
+    id: str
+    collection_id: str
+    path: str
+    content: str
+    checksum: str
+    file_size: int
+    metadata: DocumentMetadata = field(default_factory=DocumentMetadata)
+    chunks: list[DocumentChunk] = field(default_factory=list)
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    indexed_at: datetime | None = None
+
+    @property
+    def title(self) -> str | None:
+        """Get document title from metadata or path."""
+        if self.metadata.title:
+            return self.metadata.title
+        # Extract filename as fallback
+        import os  # noqa: PLC0415
+
+        return os.path.splitext(os.path.basename(self.path))[0]
+
+    @property
+    def chunk_count(self) -> int:
+        """Get the number of chunks."""
+        return len(self.chunks)
+
+    def add_chunk(self, chunk: DocumentChunk) -> None:
+        """Add a chunk to the document."""
+        self.chunks.append(chunk)
+
+    def clear_chunks(self) -> None:
+        """Remove all chunks from the document."""
+        self.chunks.clear()
+
+    def mark_indexed(self) -> None:
+        """Mark the document as indexed."""
+        self.indexed_at = datetime.now(timezone.utc)
+        self.updated_at = datetime.now(timezone.utc)
+
+
+class DocumentRepository(Protocol):
+    """Protocol for document repository operations."""
+
+    def get_by_id(self, document_id: str) -> Document | None:
+        """Get a document by ID."""
+        ...
+
+    def get_by_path(self, path: str, collection_id: str) -> Document | None:
+        """Get a document by path within a collection."""
+        ...
+
+    def list_by_collection(self, collection_id: str) -> list[Document]:
+        """List all documents in a collection."""
+        ...
+
+    def create(self, document: Document) -> Document:
+        """Create a new document."""
+        ...
+
+    def update(self, document: Document) -> Document:
+        """Update an existing document."""
+        ...
+
+    def delete(self, document_id: str) -> bool:
+        """Delete a document by ID."""
+        ...
+
+    def delete_by_collection(self, collection_id: str) -> int:
+        """Delete all documents in a collection. Returns count deleted."""
+        ...
+
+    def exists(self, path: str, collection_id: str) -> bool:
+        """Check if a document exists at the given path."""
+        ...
+
+    def get_checksum(self, document_id: str) -> str | None:
+        """Get the checksum of a document."""
+        ...

sif/core/models.py

@@ -0,0 +1,298 @@
+"""Core data models for SIF."""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from pathlib import Path
+from typing import Any, Protocol
+
+
+class ChunkStrategy(Enum):
+    """Document chunking strategies."""
+
+    FIXED = "fixed"
+    MARKDOWN = "markdown"
+    CODE = "code"
+    AUTO = "auto"
+
+
+class SearchType(Enum):
+    """Search types."""
+
+    BM25 = "bm25"
+    VECTOR = "vector"
+    HYBRID = "hybrid"
+    HYDE = "hyde"
+    EXPAND = "expand"
+
+
+@dataclass
+class Collection:
+    """A document collection."""
+
+    name: str
+    path: str
+    pattern: str = "**/*.md"
+    ignore_patterns: list[str] = field(default_factory=list)
+    include_by_default: bool = True
+    description: str | None = None
+    pre_update_cmd: str | None = None
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    document_count: int = 0
+    chunk_count: int = 0
+    last_indexed_at: datetime | None = None
+
+    def mark_indexed(self) -> None:
+        """Mark collection as indexed, updating timestamps."""
+        self.last_indexed_at = datetime.now(timezone.utc)
+        self.updated_at = datetime.now(timezone.utc)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "id": self.id,
+            "name": self.name,
+            "path": self.path,
+            "pattern": self.pattern,
+            "ignore_patterns": self.ignore_patterns,
+            "include_by_default": self.include_by_default,
+            "description": self.description,
+            "pre_update_cmd": self.pre_update_cmd,
+            "created_at": self.created_at.isoformat(),
+            "updated_at": self.updated_at.isoformat(),
+            "document_count": self.document_count,
+            "chunk_count": self.chunk_count,
+            "last_indexed_at": self.last_indexed_at.isoformat() if self.last_indexed_at else None,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Collection:
+        """Create from dictionary."""
+        return cls(
+            id=data.get("id", str(uuid.uuid4())),
+            name=data["name"],
+            path=data["path"],
+            pattern=data.get("pattern", "**/*.md"),
+            ignore_patterns=data.get("ignore_patterns", []),
+            include_by_default=data.get("include_by_default", True),
+            description=data.get("description"),
+            pre_update_cmd=data.get("pre_update_cmd"),
+            created_at=datetime.fromisoformat(data["created_at"])
+            if "created_at" in data
+            else datetime.now(timezone.utc),
+            updated_at=datetime.fromisoformat(data["updated_at"])
+            if "updated_at" in data
+            else datetime.now(timezone.utc),
+            document_count=data.get("document_count", 0),
+            chunk_count=data.get("chunk_count", 0),
+            last_indexed_at=datetime.fromisoformat(data["last_indexed_at"])
+            if data.get("last_indexed_at")
+            else None,
+        )
+
+
+@dataclass
+class DocumentChunk:
+    """A chunk of a document."""
+
+    content: str
+    sequence: int
+    start_pos: int
+    end_pos: int
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    document_id: str | None = None
+    token_count: int = 0
+    embedding: list[float] | None = None
+    embedding_id: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "id": self.id,
+            "document_id": self.document_id,
+            "content": self.content,
+            "sequence": self.sequence,
+            "start_pos": self.start_pos,
+            "end_pos": self.end_pos,
+            "token_count": self.token_count,
+            "embedding_id": self.embedding_id,
+        }
+
+
+@dataclass
+class Document:
+    """A document in the index."""
+
+    path: str
+    collection_id: str
+    content: str
+    title: str | None = None
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    filename: str = field(init=False)
+    checksum: str = field(init=False)
+    file_size: int = field(init=False)
+    mtime: float = field(default_factory=lambda: datetime.now(timezone.utc).timestamp())
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    chunks: list[DocumentChunk] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Initialize computed fields."""
+        self.filename = Path(self.path).name
+        self.checksum = hashlib.sha256(self.content.encode()).hexdigest()
+        self.file_size = len(self.content.encode())
+        if self.title is None:
+            self.title = self.filename
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "id": self.id,
+            "path": self.path,
+            "collection_id": self.collection_id,
+            "filename": self.filename,
+            "title": self.title,
+            "checksum": self.checksum,
+            "file_size": self.file_size,
+            "mtime": self.mtime,
+            "created_at": self.created_at.isoformat(),
+            "updated_at": self.updated_at.isoformat(),
+            "metadata": self.metadata,
+            "chunk_count": len(self.chunks),
+        }
+
+
+@dataclass
+class PathContext:
+    """Context information for a path."""
+
+    path: str
+    context: str
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    collection_id: str | None = None
+    context_type: str = "path"
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "id": self.id,
+            "path": self.path,
+            "collection_id": self.collection_id,
+            "context_type": self.context_type,
+            "context": self.context,
+            "created_at": self.created_at.isoformat(),
+            "updated_at": self.updated_at.isoformat(),
+        }
+
+
+@dataclass
+class SearchResult:
+    """A search result."""
+
+    document_id: str
+    title: str
+    path: str
+    collection_name: str
+    score: float
+    content: str | None = None
+    highlights: list[str] = field(default_factory=list)
+    rank: int = 0
+    scores: dict[str, float | None] = field(default_factory=dict)
+    snippet: str | None = None
+    context_description: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "document_id": self.document_id,
+            "title": self.title,
+            "path": self.path,
+            "collection_name": self.collection_name,
+            "score": self.score,
+            "content": self.content,
+            "highlights": self.highlights,
+            "rank": self.rank,
+            "scores": self.scores,
+            "snippet": self.snippet,
+            "context_description": self.context_description,
+        }
+
+
+@dataclass
+class SearchOptions:
+    """Options for search."""
+
+    limit: int = 10
+    offset: int = 0
+    collection_ids: list[str] | None = None
+    min_score: float = 0.0
+    include_content: bool = False
+    include_highlights: bool = True
+    max_highlights: int = 3
+    explain: bool = False
+    candidate_limit: int = 20
+    intent: str | None = None
+    snippet_max_length: int = 300
+
+
+@dataclass
+class IndexStats:
+    """Statistics about the index."""
+
+    collection_count: int = 0
+    document_count: int = 0
+    chunk_count: int = 0
+    total_size_bytes: int = 0
+    last_indexed_at: datetime | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "collection_count": self.collection_count,
+            "document_count": self.document_count,
+            "chunk_count": self.chunk_count,
+            "total_size_bytes": self.total_size_bytes,
+            "last_indexed_at": self.last_indexed_at.isoformat() if self.last_indexed_at else None,
+        }
+
+
+class Embedder(Protocol):
+    """Protocol for embedding models."""
+
+    def embed(self, text: str) -> list[float]:
+        """Embed a single text."""
+        ...
+
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """Embed multiple texts."""
+        ...
+
+    @property
+    def dimension(self) -> int:
+        """Get embedding dimension."""
+        ...
+
+
+class Reranker(Protocol):
+    """Protocol for reranking models."""
+
+    def rerank(self, query: str, documents: list[str]) -> list[tuple[int, float]]:
+        """Rerank documents for a query."""
+        ...
+
+
+class QueryExpander(Protocol):
+    """Protocol for query expansion."""
+
+    def expand(self, query: str) -> list[str]:
+        """Expand a query into multiple variants."""
+        ...

sif/database/__init__.py

@@ -0,0 +1,23 @@
+"""Database access layer for SIF."""
+
+from sif.database.connection import ConnectionPool, DatabaseConnection
+from sif.database.migrations import MigrationManager
+from sif.database.repositories import (
+    CollectionRepository,
+    DocumentChunkRepository,
+    DocumentRepository,
+    LLMCacheRepository,
+    PathContextRepository,
+)
+
+
+__all__ = [
+    "CollectionRepository",
+    "ConnectionPool",
+    "DatabaseConnection",
+    "DocumentChunkRepository",
+    "DocumentRepository",
+    "LLMCacheRepository",
+    "MigrationManager",
+    "PathContextRepository",
+]

sif/database/connection.py

@@ -0,0 +1,139 @@
+"""Database connection management."""
+
+import sqlite3
+from collections.abc import Generator
+from contextlib import contextmanager
+from pathlib import Path
+
+import sqlite_vec
+
+
+class DatabaseConnection:
+    """Manages SQLite database connections.
+
+    Provides connection pooling and ensures proper setup of
+    required extensions (FTS5, sqlite-vec).
+    """
+
+    def __init__(self, db_path: Path | str) -> None:
+        self._db_path = Path(db_path)
+        self._db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def _create_connection(self) -> sqlite3.Connection:
+        """Create a new database connection with extensions loaded."""
+        conn = sqlite3.connect(str(self._db_path), check_same_thread=False)
+        conn.row_factory = sqlite3.Row
+
+        # Enable foreign keys
+        conn.execute("PRAGMA foreign_keys = ON")
+
+        # Load sqlite-vec extension
+        conn.enable_load_extension(True)
+        sqlite_vec.load(conn)
+        conn.enable_load_extension(False)
+
+        return conn
+
+    @contextmanager
+    def connect(self) -> Generator[sqlite3.Connection, None, None]:
+        """Get a database connection as a context manager."""
+        conn = self._create_connection()
+        try:
+            yield conn
+        finally:
+            conn.close()
+
+    @contextmanager
+    def transaction(self) -> Generator[sqlite3.Connection, None, None]:
+        """Execute operations within a transaction."""
+        conn = self._create_connection()
+        try:
+            yield conn
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+
+    def execute(
+        self,
+        query: str,
+        parameters: tuple | dict | None = None,
+    ) -> sqlite3.Cursor:
+        """Execute a query and return the cursor."""
+        with self.connect() as conn:
+            cursor = conn.execute(query, parameters) if parameters else conn.execute(query)
+            conn.commit()
+            return cursor
+
+    def executemany(
+        self,
+        query: str,
+        parameters: list[tuple | dict],
+    ) -> sqlite3.Cursor:
+        """Execute a query multiple times."""
+        with self.connect() as conn:
+            cursor = conn.executemany(query, parameters)
+            conn.commit()
+            return cursor
+
+    def fetchone(
+        self,
+        query: str,
+        parameters: tuple | dict | None = None,
+    ) -> sqlite3.Row | None:
+        """Fetch a single row."""
+        with self.connect() as conn:
+            cursor = conn.execute(query, parameters or ())
+            return cursor.fetchone()
+
+    def fetchall(
+        self,
+        query: str,
+        parameters: tuple | dict | None = None,
+    ) -> list[sqlite3.Row]:
+        """Fetch all rows."""
+        with self.connect() as conn:
+            cursor = conn.execute(query, parameters or ())
+            return cursor.fetchall()
+
+
+class ConnectionPool:
+    """Simple connection pool for database connections.
+
+    Note: For SQLite, this is primarily useful for read-heavy workloads.
+    Writes should use a single connection to avoid locking issues.
+    """
+
+    def __init__(
+        self,
+        db_path: Path | str,
+        max_connections: int = 5,
+    ) -> None:
+        self._db_path = Path(db_path)
+        self._max_connections = max_connections
+        self._connections: list[sqlite3.Connection] = []
+
+    def get_connection(self) -> sqlite3.Connection:
+        """Get a connection from the pool or create a new one."""
+        if self._connections:
+            return self._connections.pop()
+
+        conn = sqlite3.connect(str(self._db_path), check_same_thread=False)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA foreign_keys = ON")
+        return conn
+
+    def release_connection(self, conn: sqlite3.Connection) -> None:
+        """Return a connection to the pool."""
+        if len(self._connections) < self._max_connections:
+            self._connections.append(conn)
+        else:
+            conn.close()
+
+    def close_all(self) -> None:
+        """Close all connections in the pool."""
+        for conn in self._connections:
+            conn.close()
+        self._connections.clear()