docsif 0.1.0__py3-none-any.whl

sif/search/strategy.py

@@ -0,0 +1,97 @@
+"""Search strategy interface using the Strategy pattern."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from sif.models.search import SearchOptions, SearchResult
+
+
+@dataclass
+class SearchContext:
+    """Context passed to search strategies.
+
+    Contains all necessary information for performing a search,
+    including pre-computed embeddings and context strings.
+    """
+
+    query: str
+    query_embedding: list[float] | None = None
+    expanded_query: str | None = None
+    collection_ids: list[str] | None = None
+    context_string: str = ""
+    filters: dict[str, Any] | None = None
+
+
+class SearchStrategy(ABC):
+    """Abstract base class for search strategies.
+
+    Implements the Strategy pattern to allow different search
+    algorithms to be used interchangeably.
+    """
+
+    @abstractmethod
+    def search(
+        self,
+        context: SearchContext,
+        options: SearchOptions,
+    ) -> list[SearchResult]:
+        """Execute search and return results.
+
+        Args:
+            context: Search context with query and embeddings
+            options: Search options and parameters
+
+        Returns:
+            List of search results ordered by relevance
+        """
+        ...
+
+    @abstractmethod
+    def search_batch(
+        self,
+        contexts: list[SearchContext],
+        options: SearchOptions,
+    ) -> list[list[SearchResult]]:
+        """Execute batch search.
+
+        Args:
+            contexts: List of search contexts
+            options: Search options and parameters
+
+        Returns:
+            List of search results for each context
+        """
+        ...
+
+
+class Searchable(Protocol):
+    """Protocol for searchable repositories."""
+
+    def search_fts(
+        self,
+        query: str,
+        collection_ids: list[str] | None,
+        limit: int,
+        offset: int,
+    ) -> list[tuple[str, float]]:
+        """Search using full-text search.
+
+        Returns:
+            List of (document_id, score) tuples
+        """
+        ...
+
+    def search_vector(
+        self,
+        embedding: list[float],
+        collection_ids: list[str] | None,
+        limit: int,
+        offset: int,
+    ) -> list[tuple[str, float]]:
+        """Search using vector similarity.
+
+        Returns:
+            List of (document_id, score) tuples
+        """
+        ...

sif/search/vector.py

@@ -0,0 +1,177 @@
+"""Vector similarity search implementation."""
+
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+
+from sif.core.models import SearchOptions, SearchResult
+
+
+class VectorSearcher:
+    """Vector similarity search using sqlite-vec or fallback."""
+
+    def __init__(self, db: sqlite3.Connection, embedding_dim: int = 768) -> None:
+        self.db = db
+        self.embedding_dim = embedding_dim
+        self._vec_available = self._check_vec_extension()
+        if not self._vec_available:
+            raise RuntimeError(
+                "sqlite-vec extension is not available. Install sqlite-vec to use vector search."
+            )
+
+    def _check_vec_extension(self) -> bool:
+        """Check if sqlite-vec extension is available."""
+        try:
+            self.db.execute("SELECT vec_version()")
+            return True
+        except sqlite3.OperationalError:
+            return False
+
+    def search(
+        self,
+        query_embedding: list[float],
+        options: SearchOptions | None = None,
+    ) -> list[SearchResult]:
+        """Search documents by vector similarity."""
+        if options is None:
+            options = SearchOptions()
+        return self._search_with_vec(query_embedding, options)
+
+    def _search_with_vec(
+        self,
+        query_embedding: list[float],
+        options: SearchOptions,
+    ) -> list[SearchResult]:
+        """Search using sqlite-vec."""
+        # Convert embedding to vec format
+        embedding_str = self._embedding_to_vec(query_embedding)
+
+        # Build collection filter
+        collection_filter = ""
+        params = [embedding_str]
+
+        if options.collection_ids:
+            placeholders = ", ".join(["?"] * len(options.collection_ids))
+            collection_filter = f"AND d.collection_id IN ({placeholders})"
+            params.extend(options.collection_ids)
+
+        sql = f"""
+            SELECT
+                d.id as document_id,
+                d.title,
+                d.path,
+                c.name as collection_name,
+                distance as score
+            FROM document_embeddings de
+            JOIN documents d ON de.document_id = d.id
+            JOIN collections c ON d.collection_id = c.id
+            WHERE embedding MATCH ? AND k = {options.limit} {collection_filter}
+            ORDER BY distance
+        """
+
+        cursor = self.db.execute(sql, params)
+        results = []
+
+        for rank, row in enumerate(cursor.fetchall(), 1):
+            # Convert distance to score (lower distance = higher score)
+            # Cosine distance is 0-2, convert to 0-1 score
+            score = 1.0 - (row["score"] / 2.0)
+
+            if score < options.min_score:
+                continue
+
+            result = SearchResult(
+                document_id=row["document_id"],
+                title=row["title"] or "",
+                path=row["path"],
+                collection_name=row["collection_name"],
+                score=score,
+                rank=rank,
+            )
+
+            if options.include_content:
+                result.content = self._get_document_content(row["document_id"])
+
+            results.append(result)
+
+        return self._attach_contexts(results)
+
+    def _attach_contexts(self, results: list[SearchResult]) -> list[SearchResult]:
+        """Attach path context descriptions to search results via batch query."""
+        if not results:
+            return results
+        paths = list({r.path for r in results})
+        placeholders = ", ".join(["?"] * len(paths))
+        sql = f"""
+            SELECT target_id, content FROM contexts
+            WHERE context_type = 'path' AND target_id IN ({placeholders})
+        """
+        cursor = self.db.execute(sql, paths)
+        # Normalize keys for cross-platform matching (macOS /private/tmp, etc.)
+        context_map = {
+            os.path.realpath(row["target_id"]): row["content"] for row in cursor.fetchall()
+        }
+        for result in results:
+            result.context_description = context_map.get(os.path.realpath(result.path))
+        return results
+
+    def _embedding_to_vec(self, embedding: list[float]) -> str:
+        """Convert embedding to sqlite-vec format."""
+        return json.dumps(embedding)
+
+    def _get_document_content(self, document_id: str) -> str | None:
+        """Get document content."""
+        cursor = self.db.execute("SELECT content FROM documents WHERE id = ?", (document_id,))
+        row = cursor.fetchone()
+        return row[0] if row else None
+
+    def add_embedding(
+        self,
+        embedding_id: str,
+        document_id: str,
+        chunk_id: str | None,
+        embedding: list[float],
+    ) -> None:
+        """Add an embedding to the index."""
+        self._add_embedding_vec(embedding_id, document_id, chunk_id, embedding)
+
+    def _add_embedding_vec(
+        self,
+        embedding_id: str,
+        document_id: str,
+        chunk_id: str | None,
+        embedding: list[float],
+    ) -> None:
+        """Add embedding using sqlite-vec."""
+        embedding_str = self._embedding_to_vec(embedding)
+
+        self.db.execute(
+            """
+            INSERT OR REPLACE INTO document_embeddings
+            (embedding_id, document_id, chunk_id, embedding)
+            VALUES (?, ?, ?, vec_f32(?))
+            """,
+            (embedding_id, document_id, chunk_id, embedding_str),
+        )
+
+    def add_embeddings_batch(
+        self,
+        items: list[tuple[str, str, str | None, list[float]]],
+    ) -> None:
+        """Add multiple embeddings in a single batch operation.
+
+        Each item is a tuple of (embedding_id, document_id, chunk_id, embedding_vector).
+        """
+        if not items:
+            return
+        rows = [(eid, doc_id, chunk_id, json.dumps(vec)) for eid, doc_id, chunk_id, vec in items]
+        self.db.executemany(
+            """
+            INSERT OR REPLACE INTO document_embeddings
+            (embedding_id, document_id, chunk_id, embedding)
+            VALUES (?, ?, ?, vec_f32(?))
+            """,
+            rows,
+        )

sif/utils/__init__.py

@@ -0,0 +1,26 @@
+"""Utility functions and helpers for SIF.
+
+This package provides various utility functions:
+- Logging configuration
+- File path utilities
+- Text processing helpers
+- Progress indicators
+"""
+
+from sif.utils.logging import get_logger, setup_logging
+from sif.utils.paths import get_cache_dir, get_config_dir, get_data_dir
+from sif.utils.progress import ProgressTracker
+from sif.utils.text import count_tokens, normalize_text, truncate_text
+
+
+__all__ = [
+    "ProgressTracker",
+    "count_tokens",
+    "get_cache_dir",
+    "get_config_dir",
+    "get_data_dir",
+    "get_logger",
+    "normalize_text",
+    "setup_logging",
+    "truncate_text",
+]

sif/utils/logging.py

@@ -0,0 +1,143 @@
+"""Logging utilities."""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import os
+import sys
+import warnings
+
+
+# Default logging format
+DEFAULT_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+SIMPLE_FORMAT = "%(levelname)s: %(message)s"
+
+# Noisy third-party loggers to suppress in quiet mode
+_NOISY_LOGGERS = [
+    "modelscope",
+    "transformers",
+    "sentence_transformers",
+    "torch",
+    "urllib3",
+    "huggingface_hub",
+    "auto_gptq",
+    "bitsandbytes",
+]
+
+# Global quiet mode flag
+_quiet_mode = False
+
+
+def is_quiet() -> bool:
+    """Check if quiet mode is enabled."""
+    return _quiet_mode
+
+
+def set_quiet(quiet: bool = True) -> None:
+    """Set quiet mode globally.
+
+    Use this when quiet mode is triggered after setup_logging has already run.
+    """
+    global _quiet_mode
+    _quiet_mode = quiet
+    if quiet:
+        _apply_quiet_mode()
+
+
+def _apply_quiet_mode() -> None:
+    """Apply quiet mode side effects (logging, env vars, warnings)."""
+    logging.disable(logging.INFO)
+    for name in _NOISY_LOGGERS:
+        lib_logger = logging.getLogger(name)
+        lib_logger.setLevel(logging.WARNING)
+        for h in lib_logger.handlers[:]:
+            lib_logger.removeHandler(h)
+        lib_logger.propagate = False
+
+    os.environ.setdefault("TQDM_DISABLE", "1")
+    os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
+    os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
+
+    try:
+        from transformers import logging as transformers_logging  # noqa: PLC0415
+
+        transformers_logging.set_verbosity_error()
+    except ImportError:
+        pass
+
+    warnings.filterwarnings("ignore")
+
+
+@contextlib.contextmanager
+def suppress_output():
+    """Suppress both stdout and stderr output within the context block."""
+    with open(os.devnull, "w") as devnull:
+        with contextlib.redirect_stdout(devnull), contextlib.redirect_stderr(devnull):
+            yield
+
+
+@contextlib.contextmanager
+def suppress_stdout():
+    """Suppress stdout output within the context block."""
+    with open(os.devnull, "w") as devnull, contextlib.redirect_stdout(devnull):
+        yield
+
+
+@contextlib.contextmanager
+def suppress_stderr():
+    """Suppress stderr output within the context block."""
+    with open(os.devnull, "w") as devnull, contextlib.redirect_stderr(devnull):
+        yield
+
+
+def setup_logging(
+    level: str = "INFO",
+    format_str: str | None = None,
+    handler: logging.Handler | None = None,
+) -> None:
+    """Setup logging configuration.
+
+    Args:
+        level: Logging level (DEBUG, INFO, WARNING, ERROR)
+        format_str: Log format string
+        handler: Custom log handler
+    """
+    global _quiet_mode
+    _quiet_mode = level.upper() == "WARNING"
+
+    if format_str is None:
+        format_str = SIMPLE_FORMAT if level in ("INFO", "WARNING") else DEFAULT_FORMAT
+
+    # Create formatter
+    formatter = logging.Formatter(format_str)
+
+    # Create handler if not provided
+    if handler is None:
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(formatter)
+
+    # Configure root logger
+    root_logger = logging.getLogger("sif")
+    root_logger.setLevel(getattr(logging, level.upper()))
+    root_logger.handlers = []
+    root_logger.addHandler(handler)
+
+    # Don't propagate to parent
+    root_logger.propagate = False
+
+    # In quiet mode, suppress noisy third-party libraries and progress bars
+    if _quiet_mode:
+        _apply_quiet_mode()
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Get a logger instance.
+
+    Args:
+        name: Logger name (typically __name__)
+
+    Returns:
+        Logger instance
+    """
+    return logging.getLogger(f"sif.{name}")

sif/utils/paths.py

@@ -0,0 +1,82 @@
+"""Path utilities for SIF."""
+
+from pathlib import Path
+
+from platformdirs import user_cache_dir, user_config_dir, user_data_dir
+
+from sif.config.constants import APP_NAME
+
+
+def get_data_dir() -> Path:
+    """Get the application data directory.
+
+    Returns:
+        Path to the data directory
+    """
+    path = Path(user_data_dir(APP_NAME))
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def get_cache_dir() -> Path:
+    """Get the application cache directory.
+
+    Returns:
+        Path to the cache directory
+    """
+    path = Path(user_cache_dir(APP_NAME))
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def get_config_dir() -> Path:
+    """Get the application config directory.
+
+    Returns:
+        Path to the config directory
+    """
+    path = Path(user_config_dir(APP_NAME))
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def expand_path(path: str | Path) -> Path:
+    """Expand a path, resolving ~ and environment variables.
+
+    Args:
+        path: Path to expand
+
+    Returns:
+        Expanded absolute path
+    """
+    return Path(path).expanduser().resolve()
+
+
+def is_markdown_file(path: Path) -> bool:
+    """Check if a file is a markdown file.
+
+    Args:
+        path: File path to check
+
+    Returns:
+        True if the file has a markdown extension
+    """
+    from sif.config.constants import MARKDOWN_EXTENSIONS  # noqa: PLC0415
+
+    return path.suffix.lower() in MARKDOWN_EXTENSIONS
+
+
+def get_relative_path(path: Path, base: Path) -> Path:
+    """Get path relative to a base directory.
+
+    Args:
+        path: Absolute path
+        base: Base directory
+
+    Returns:
+        Relative path from base to path
+    """
+    try:
+        return path.relative_to(base)
+    except ValueError:
+        return path

sif/utils/progress.py

@@ -0,0 +1,83 @@
+"""Progress tracking utilities."""
+
+from typing import Protocol
+
+from rich.progress import BarColumn, Progress as RichProgress, SpinnerColumn, TaskID, TextColumn
+
+from sif.utils.logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class ProgressTracker(Protocol):
+    """Protocol for progress tracking."""
+
+    def start(self, description: str, total: int | None = None) -> None:
+        """Start tracking progress."""
+        ...
+
+    def update(self, advance: int = 1) -> None:
+        """Update progress."""
+        ...
+
+    def finish(self) -> None:
+        """Finish tracking."""
+        ...
+
+
+class RichProgressTracker:
+    """Progress tracker using Rich library."""
+
+    def __init__(self) -> None:
+        """Initialize progress tracker."""
+        self._progress: RichProgress | None = None
+        self._task_id: TaskID | None = None
+
+    def start(self, description: str, total: int | None = None) -> None:
+        """Start tracking progress.
+
+        Args:
+            description: Task description
+            total: Total items (None for indeterminate)
+        """
+        self._progress = RichProgress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn() if total else None,
+            TextColumn("[progress.percentage]{task.percentage:>3.0f}%") if total else None,
+        )
+        self._progress.start()
+        self._task_id = self._progress.add_task(description, total=total)
+
+    def update(self, advance: int = 1) -> None:
+        """Update progress.
+
+        Args:
+            advance: Number of items to advance
+        """
+        if self._progress and self._task_id is not None:
+            self._progress.update(self._task_id, advance=advance)
+
+    def finish(self) -> None:
+        """Finish tracking."""
+        if self._progress:
+            self._progress.stop()
+            self._progress = None
+            self._task_id = None
+
+
+class NullProgressTracker:
+    """No-op progress tracker for silent operation."""
+
+    def start(self, description: str, total: int | None = None) -> None:
+        """Start tracking (no-op)."""
+        pass
+
+    def update(self, advance: int = 1) -> None:
+        """Update progress (no-op)."""
+        pass
+
+    def finish(self) -> None:
+        """Finish tracking (no-op)."""
+        pass