docsif 0.1.0__py3-none-any.whl

sif/cli/commands/pull.py

@@ -0,0 +1,111 @@
+"""Model download command."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from urllib import request
+
+import click
+from rich.console import Console
+
+
+try:
+    from huggingface_hub import hf_hub_download
+except ImportError:  # pragma: no cover
+    hf_hub_download = None  # type: ignore[misc,assignment]
+
+try:
+    from modelscope import snapshot_download
+except ImportError:  # pragma: no cover
+    snapshot_download = None  # type: ignore[misc,assignment]
+
+console = Console()
+
+_MIN_SPEC_PARTS = 3
+
+
+def _download_from_url(model_spec: str, target_cache: Path) -> Path:
+    local_name = Path(model_spec).name
+    if not local_name:
+        raise click.ClickException("Could not determine filename from URL")
+    downloaded = target_cache / local_name
+    console.print(f"[dim]Downloading from URL: {model_spec}[/dim]")
+    try:
+        request.urlretrieve(model_spec, downloaded)
+    except Exception as e:
+        raise click.ClickException(f"Download failed: {e}") from e
+    return downloaded
+
+
+def _download_from_hf(repo_id: str, filename: str, target_cache: Path) -> Path:
+    if hf_hub_download is None:
+        raise ImportError("huggingface_hub not installed")
+
+    console.print(f"[dim]Downloading from HuggingFace: {repo_id}/{filename}[/dim]")
+    path = hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        cache_dir=str(target_cache),
+        local_files_only=False,
+    )
+    return Path(path)
+
+
+def _download_from_modelscope(repo_id: str, filename: str, target_cache: Path) -> Path:
+    if snapshot_download is None:
+        raise ImportError("modelscope not installed")
+
+    console.print("[dim]Trying ModelScope fallback...[/dim]")
+    model_path = snapshot_download(repo_id, cache_dir=str(target_cache))
+    return Path(model_path) / filename
+
+
+def _download_model(model_spec: str, target_cache: Path) -> Path:
+    if model_spec.startswith(("http://", "https://")):
+        return _download_from_url(model_spec, target_cache)
+
+    parts = model_spec.split("/")
+    if len(parts) < _MIN_SPEC_PARTS:
+        raise click.ClickException("Model spec must be owner/repo/filename.gguf or a direct URL")
+    repo_id = "/".join(parts[:-1])
+    filename = parts[-1]
+
+    try:
+        return _download_from_hf(repo_id, filename, target_cache)
+    except Exception as hf_err:
+        console.print(f"[yellow]HuggingFace failed: {hf_err}[/yellow]")
+        try:
+            return _download_from_modelscope(repo_id, filename, target_cache)
+        except ImportError:
+            raise click.ClickException(
+                f"HuggingFace failed: {hf_err}. ModelScope not installed."
+            ) from hf_err
+        except Exception as ms_err:
+            raise click.ClickException(f"Download failed: HF={hf_err}, MS={ms_err}") from ms_err
+
+
+@click.command("pull")
+@click.argument("model_spec")
+@click.option(
+    "--cache-dir",
+    type=click.Path(),
+    help="Custom cache directory",
+)
+@click.pass_context
+def pull_cmd(
+    _ctx: click.Context,
+    model_spec: str,
+    cache_dir: str | None,
+) -> None:
+    """Download a GGUF model file."""
+    target_cache = (
+        Path(cache_dir).expanduser().resolve() if cache_dir else Path.home() / ".sif" / "models"
+    )
+    target_cache.mkdir(parents=True, exist_ok=True)
+
+    downloaded = _download_model(model_spec, target_cache)
+
+    if not downloaded.exists() or downloaded.stat().st_size == 0:
+        raise click.ClickException("Downloaded file is missing or empty.")
+
+    console.print(f"[green]Downloaded to {downloaded}[/green]")

sif/cli/commands/search.py

@@ -0,0 +1,574 @@
+"""Search commands."""
+
+from __future__ import annotations
+
+import json
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from sif.cli.formatters import add_line_numbers_to_results, prepend_line_numbers
+from sif.core.models import SearchOptions
+from sif.database.database import Database
+from sif.database.repositories import CollectionRepository
+from sif.search.bm25 import BM25Searcher
+from sif.search.hybrid import SearchPipeline
+from sif.utils.logging import set_quiet
+
+
+console = Console()
+
+
+def format_results_json(results: list) -> str:
+    """Format results as JSON."""
+    return json.dumps(
+        [r.to_dict() if hasattr(r, "to_dict") else r for r in results],
+        indent=2,
+        ensure_ascii=False,
+    )
+
+
+def format_results_csv(results: list) -> str:
+    """Format results as CSV."""
+    lines = ["rank,score,title,path,collection"]
+    lines.extend(
+        f'"{r.rank}","{r.score:.4f}","{r.title}","{r.path}","{r.collection_name}"' for r in results
+    )
+    return "\n".join(lines)
+
+
+def format_results_md(results: list, query: str) -> str:
+    """Format results as Markdown."""
+    lines = [f"# Search Results: {query}", ""]
+    for r in results:
+        lines.append(f"## {r.rank}. {r.title}")
+        lines.append(f"- **Score:** {r.score:.4f}")
+        lines.append(f"- **Path:** `{r.path}`")
+        lines.append(f"- **Collection:** {r.collection_name}")
+        if hasattr(r, "get") and r.get("line_numbers"):
+            lines.append(f"- **Line Numbers:** `{r.get('line_numbers')}`")
+        elif hasattr(r, "line_numbers") and r.line_numbers:
+            lines.append(f"- **Line Numbers:** `{r.line_numbers}`")
+        if hasattr(r, "highlights") and r.highlights:
+            lines.append("")
+            lines.append("> " + "\n> ".join(r.highlights[:3]))
+        lines.append("")
+    return "\n".join(lines)
+
+
+def format_results_xml(results: list, query: str) -> str:
+    """Format results as XML."""
+    lines = ['<?xml version="1.0" encoding="UTF-8"?>', f'<search query="{query}">']
+    for r in results:
+        lines.append(f'  <result rank="{r.rank}" score="{r.score:.4f}">')
+        lines.append(f"    <title>{r.title}</title>")
+        lines.append(f"    <path>{r.path}</path>")
+        lines.append(f"    <collection>{r.collection_name}</collection>")
+        if hasattr(r, "get") and r.get("line_numbers"):
+            lines.append(f"    <line_numbers>{r.get('line_numbers')}</line_numbers>")
+        elif hasattr(r, "line_numbers") and r.line_numbers:
+            lines.append(f"    <line_numbers>{r.line_numbers}</line_numbers>")
+        lines.append("  </result>")
+    lines.append("</search>")
+    return "\n".join(lines)
+
+
+@click.group("search")
+def search_group() -> None:
+    """Search commands."""
+    pass
+
+
+@search_group.command("search")
+@click.argument("query")
+@click.option("-n", "--limit", default=10, help="Number of results")
+@click.option("-c", "--collection", multiple=True, help="Collection to search")
+@click.option("--all", "search_all", is_flag=True, help="Search all collections")
+@click.option("--min-score", default=0.0, help="Minimum score threshold")
+@click.option("--full", is_flag=True, help="Include full content")
+@click.option("--explain", is_flag=True, help="Show search explanation")
+@click.option("--line-numbers", is_flag=True, help="Show line numbers in content")
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
+@click.option("--csv", "output_csv", is_flag=True, help="Output as CSV")
+@click.option("--md", "output_md", is_flag=True, help="Output as Markdown")
+@click.option("--xml", "output_xml", is_flag=True, help="Output as XML")
+@click.option("--files", "output_files", is_flag=True, help="Output file paths only")
+@click.option("-q", "--quiet", is_flag=True, help="Suppress non-error output")
+@click.pass_context
+def search_cmd(  # noqa: C901, PLR0912, PLR0913, PLR0915
+    ctx: click.Context,
+    query: str,
+    limit: int,
+    collection: tuple,
+    search_all: bool,
+    min_score: float,
+    full: bool,
+    explain: bool,
+    line_numbers: bool,
+    output_json: bool,
+    output_csv: bool,
+    output_md: bool,
+    output_xml: bool,
+    output_files: bool,
+    quiet: bool,
+) -> None:
+    """Search documents using BM25."""
+    index_path = ctx.obj["index_path"]
+    quiet = quiet or ctx.obj.get("quiet", False)
+    if quiet:
+        set_quiet(True)
+
+    if not index_path.exists():
+        if not quiet:
+            console.print("[yellow]No index found. Run 'sif update' first.[/yellow]")
+        return
+
+    db = Database(index_path)
+    db.init_schema()
+
+    # Build options
+    options = SearchOptions(
+        limit=limit,
+        min_score=min_score,
+        include_content=full,
+        include_highlights=True,
+    )
+
+    # Get collection IDs if specified
+    if collection:
+        with db.connection:
+            repo = CollectionRepository(db.connection)
+            options.collection_ids = []
+            for name in collection:
+                coll = repo.get_by_name(name)
+                if coll:
+                    options.collection_ids.append(coll.id)
+    elif not search_all:
+        with db.connection:
+            repo = CollectionRepository(db.connection)
+            enabled = repo.list_enabled()
+            options.collection_ids = [c.id for c in enabled]
+
+    # Search
+    with db.connection:
+        searcher = BM25Searcher(db.connection)
+        results = searcher.search(query, options)
+
+    # Output results
+    quiet = quiet or ctx.obj.get("quiet", False)
+    if output_files or (quiet and not any([output_json, output_csv, output_md, output_xml])):
+        for r in results:
+            console.print(r.path)
+    elif output_json:
+        console.print(
+            format_results_json(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results
+            )
+        )
+    elif output_csv:
+        console.print(
+            format_results_csv(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results
+            )
+        )
+    elif output_md:
+        console.print(
+            format_results_md(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results,
+                query,
+            )
+        )
+    elif output_xml:
+        console.print(
+            format_results_xml(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results,
+                query,
+            )
+        )
+    else:
+        # Rich table output
+        if not results:
+            if not quiet:
+                console.print("[yellow]No results found.[/yellow]")
+            return
+
+        table = Table(title=f'Search Results: "{query}"')
+        table.add_column("#", style="cyan", justify="right")
+        table.add_column("Score", style="green", justify="right")
+        table.add_column("Title", style="yellow")
+        table.add_column("Collection", style="blue")
+        if line_numbers and any(getattr(r, "content", None) for r in results):
+            table.add_column("Content", style="white")
+
+        for r in results:
+            row = [
+                str(r.rank),
+                f"{r.score:.4f}",
+                r.title[:50] + "..." if len(r.title) > 50 else r.title,
+                r.collection_name,
+            ]
+            if line_numbers and getattr(r, "content", None):
+                content = prepend_line_numbers(r.content)
+                content = content[:200] + "..." if len(content) > 200 else content
+                row.append(content)
+            table.add_row(*row)
+
+        console.print(table)
+
+        if explain:
+            console.print(f"\n[dim]Query: {query}[/dim]")
+            console.print(f"[dim]Results: {len(results)}[/dim]")
+
+
+@search_group.command("vsearch")
+@click.argument("query")
+@click.option("-n", "--limit", default=10, help="Number of results")
+@click.option("-c", "--collection", multiple=True, help="Collection to search")
+@click.option("--all", "search_all", is_flag=True, help="Search all collections")
+@click.option("--min-score", default=0.0, help="Minimum score threshold")
+@click.option("--full", is_flag=True, help="Include full content")
+@click.option("--line-numbers", is_flag=True, help="Show line numbers in content")
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
+@click.option(
+    "--model-type",
+    type=click.Choice(["sentence_transformers", "gguf", "openai", "modelscope"]),
+    help="Embedding model type override",
+)
+@click.option("-q", "--quiet", is_flag=True, help="Suppress non-error output")
+@click.pass_context
+def vsearch_cmd(  # noqa: C901, PLR0912, PLR0913, PLR0915
+    ctx: click.Context,
+    query: str,
+    limit: int,
+    collection: tuple,
+    search_all: bool,
+    min_score: float,
+    full: bool,
+    line_numbers: bool,
+    output_json: bool,
+    model_type: str | None = None,
+    quiet: bool = False,
+) -> None:
+    """Search documents using vector similarity."""
+    index_path = ctx.obj["index_path"]
+    quiet = quiet or ctx.obj.get("quiet", False)
+    if quiet:
+        set_quiet(True)
+
+    if not index_path.exists():
+        if not quiet:
+            console.print(
+                "[yellow]No index found. Run 'sif update' and 'sif embed' first.[/yellow]"
+            )
+        return
+
+    from sif.config.settings import get_settings  # noqa: PLC0415
+    from sif.database.database import Database  # noqa: PLC0415
+    from sif.database.repositories import CollectionRepository  # noqa: PLC0415
+    from sif.embedding.manager import EmbeddingManager  # noqa: PLC0415
+    from sif.search.vector import VectorSearcher  # noqa: PLC0415
+
+    settings = get_settings()
+    if model_type:
+        settings = settings.model_copy(update={"model_type": model_type})
+
+    try:
+        manager = EmbeddingManager.from_settings(settings)
+        query_embedding = manager.embed_single(query)
+    except ImportError as e:
+        raise click.ClickException(f"Embedding backend not installed: {e}") from None
+    except Exception as e:
+        raise click.ClickException(f"Failed to load embedding model: {e}") from e
+
+    db = Database(index_path)
+    db.init_schema()
+
+    options = SearchOptions(
+        limit=limit,
+        min_score=min_score,
+        include_content=full,
+        include_highlights=True,
+    )
+
+    if collection:
+        with db.connection:
+            repo = CollectionRepository(db.connection)
+            options.collection_ids = []
+            for name in collection:
+                coll = repo.get_by_name(name)
+                if coll:
+                    options.collection_ids.append(coll.id)
+    elif not search_all:
+        with db.connection:
+            repo = CollectionRepository(db.connection)
+            enabled = repo.list_enabled()
+            options.collection_ids = [c.id for c in enabled]
+
+    with db.connection:
+        searcher = VectorSearcher(db.connection, len(query_embedding))
+        results = searcher.search(query_embedding, options)
+
+    quiet = quiet or ctx.obj.get("quiet", False)
+    if quiet and not output_json:
+        for r in results:
+            console.print(r.path)
+    elif output_json:
+        console.print(
+            format_results_json(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results
+            )
+        )
+    else:
+        if not results:
+            if not quiet:
+                console.print("[yellow]No results found.[/yellow]")
+            return
+
+        table = Table(title=f'Vector Search Results: "{query}"')
+        table.add_column("#", style="cyan", justify="right")
+        table.add_column("Score", style="green", justify="right")
+        table.add_column("Title", style="yellow")
+        table.add_column("Collection", style="blue")
+        if line_numbers and any(getattr(r, "content", None) for r in results):
+            table.add_column("Content", style="white")
+
+        for r in results:
+            row = [
+                str(r.rank),
+                f"{r.score:.4f}",
+                r.title[:50] + "..." if len(r.title) > 50 else r.title,
+                r.collection_name,
+            ]
+            if line_numbers and getattr(r, "content", None):
+                content = prepend_line_numbers(r.content)
+                content = content[:200] + "..." if len(content) > 200 else content
+                row.append(content)
+            table.add_row(*row)
+
+        console.print(table)
+
+
+@search_group.command("query")
+@click.argument("query")
+@click.option("-n", "--limit", default=10, help="Number of results")
+@click.option("-c", "--collection", multiple=True, help="Collection to search")
+@click.option("--all", "search_all", is_flag=True, help="Search all collections")
+@click.option("--min-score", default=0.0, help="Minimum score threshold")
+@click.option("--full", is_flag=True, help="Include full content")
+@click.option("--explain", is_flag=True, help="Show score breakdowns across pipeline stages")
+@click.option(
+    "-C",
+    "--candidate-limit",
+    default=20,
+    type=click.IntRange(1, 200),
+    help="Reranker candidate pool size",
+)
+@click.option("--intent", help="Search intent hint for query expansion")
+@click.option("--line-numbers", is_flag=True, help="Show line numbers in content")
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
+@click.option("--csv", "output_csv", is_flag=True, help="Output as CSV")
+@click.option("--md", "output_md", is_flag=True, help="Output as Markdown")
+@click.option("--xml", "output_xml", is_flag=True, help="Output as XML")
+@click.option("--files", "output_files", is_flag=True, help="Output file paths only")
+@click.option(
+    "--model-type",
+    type=click.Choice(["sentence_transformers", "gguf", "openai", "modelscope"]),
+    help="Embedding model type override",
+)
+@click.option("-q", "--quiet", is_flag=True, help="Suppress non-error output")
+@click.pass_context
+def query_cmd(  # noqa: C901, PLR0912, PLR0913, PLR0915
+    ctx: click.Context,
+    query: str,
+    limit: int,
+    collection: tuple,
+    search_all: bool,
+    min_score: float,
+    full: bool,
+    explain: bool,
+    candidate_limit: int,
+    intent: str | None,
+    line_numbers: bool,
+    output_json: bool,
+    output_csv: bool,
+    output_md: bool,
+    output_xml: bool,
+    output_files: bool,
+    model_type: str | None = None,
+    quiet: bool = False,
+) -> None:
+    """Search documents using hybrid approach (BM25 + Vector + RRF).
+
+    This is the recommended search command for best results.
+    """
+    index_path = ctx.obj["index_path"]
+    quiet = quiet or ctx.obj.get("quiet", False)
+    if quiet:
+        set_quiet(True)
+
+    if not index_path.exists():
+        if not quiet:
+            console.print("[yellow]No index found. Run 'sif update' first.[/yellow]")
+        return
+
+    db = Database(index_path)
+    db.init_schema()
+
+    # Build options
+    options = SearchOptions(
+        limit=limit,
+        min_score=min_score,
+        include_content=full,
+        include_highlights=True,
+        explain=explain,
+        candidate_limit=candidate_limit,
+        intent=intent,
+    )
+
+    # Get collection IDs if specified
+    if collection:
+        with db.connection:
+            repo = CollectionRepository(db.connection)
+            options.collection_ids = []
+            for name in collection:
+                coll = repo.get_by_name(name)
+                if coll:
+                    options.collection_ids.append(coll.id)
+    elif not search_all:
+        with db.connection:
+            repo = CollectionRepository(db.connection)
+            enabled = repo.list_enabled()
+            options.collection_ids = [c.id for c in enabled]
+
+    # Load embedder for hybrid search
+    from sif.config.settings import get_settings  # noqa: PLC0415
+    from sif.embedding.manager import EmbeddingManager  # noqa: PLC0415
+    from sif.search.expansion import QueryExpansion  # noqa: PLC0415
+    from sif.search.rerank import create_reranker  # noqa: PLC0415
+    from sif.search.snippets import SmartSnippetExtractor  # noqa: PLC0415
+
+    settings = get_settings()
+    if model_type:
+        settings = settings.model_copy(update={"model_type": model_type})
+
+    try:
+        manager = EmbeddingManager.from_settings(settings)
+        manager.load_model()
+        embedding_dim = len(manager.embed_single("probe"))
+    except ImportError as e:
+        raise click.ClickException(f"Embedding backend not installed: {e}") from None
+    except Exception as e:
+        raise click.ClickException(f"Failed to load embedding model: {e}") from e
+    # Create optional components
+    query_expander = QueryExpansion(embedding_manager=manager)
+
+    reranker = None
+    if settings.reranker_model_name or settings.reranker_model_path:
+        try:
+            reranker = create_reranker(settings)
+        except ImportError:
+            if not quiet:
+                console.print(
+                    "[yellow]Reranker not available. "
+                    "Install with: pip install -e '.[embed]'[/yellow]"
+                )
+
+    snippet_extractor = SmartSnippetExtractor(max_length=options.snippet_max_length)
+
+    # Search using pipeline
+    with db.connection:
+        pipeline = SearchPipeline(
+            db.connection,
+            embedder=manager._model,  # noqa: SLF001
+            query_expander=query_expander,
+            reranker=reranker,
+            snippet_extractor=snippet_extractor,
+            embedding_dim=embedding_dim,
+        )
+        results = pipeline.search(query, options)
+
+    # Output results
+    quiet = quiet or ctx.obj.get("quiet", False)
+    if output_files or (quiet and not any([output_json, output_csv, output_md, output_xml])):
+        for r in results:
+            console.print(r.path)
+    elif output_json:
+        console.print(
+            format_results_json(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results
+            )
+        )
+    elif output_csv:
+        console.print(
+            format_results_csv(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results
+            )
+        )
+    elif output_md:
+        console.print(
+            format_results_md(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results,
+                query,
+            )
+        )
+    elif output_xml:
+        console.print(
+            format_results_xml(
+                add_line_numbers_to_results([r.to_dict() for r in results])
+                if line_numbers
+                else results,
+                query,
+            )
+        )
+    else:
+        # Rich table output
+        if not results:
+            if not quiet:
+                console.print("[yellow]No results found.[/yellow]")
+            return
+
+        table = Table(title=f'Hybrid Search Results: "{query}"')
+        table.add_column("#", style="cyan", justify="right")
+        table.add_column("Score", style="green", justify="right")
+        table.add_column("Title", style="yellow")
+        table.add_column("Collection", style="blue")
+        if line_numbers and any(getattr(r, "content", None) for r in results):
+            table.add_column("Content", style="white")
+
+        for r in results:
+            row = [
+                str(r.rank),
+                f"{r.score:.4f}",
+                r.title[:50] + "..." if len(r.title) > 50 else r.title,
+                r.collection_name,
+            ]
+            if line_numbers and getattr(r, "content", None):
+                content = prepend_line_numbers(r.content)
+                content = content[:200] + "..." if len(content) > 200 else content
+                row.append(content)
+            table.add_row(*row)
+
+        console.print(table)
+
+        if explain:
+            for r in results:
+                scores_str = ", ".join(f"{k}={v:.4f}" for k, v in r.scores.items() if v is not None)
+                if scores_str:
+                    console.print(f"[dim]{r.title}: {scores_str}[/dim]")