docsif 0.1.0__py3-none-any.whl

sif/cli/commands/get.py

@@ -0,0 +1,200 @@
+"""Document retrieval commands."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+from rich.console import Console
+
+from sif.cli.formatters import prepend_line_numbers
+from sif.core.models import Document
+from sif.database.database import Database
+from sif.database.repositories import DocumentRepository
+
+
+console = Console()
+
+
+@click.group("get")
+def get_group() -> None:
+    """Retrieve documents."""
+    pass
+
+
+@get_group.command("get")
+@click.argument("path_or_docid")
+@click.option("--from-line", "-f", type=int, help="Start from line number")
+@click.option("--lines", "-l", type=int, help="Number of lines to show")
+@click.option("--line-numbers", is_flag=True, help="Show line numbers")
+@click.pass_context
+def get_cmd(  # noqa: C901, PLR0912
+    ctx: click.Context,
+    path_or_docid: str,
+    from_line: int | None,
+    lines: int | None,
+    line_numbers: bool,
+) -> None:
+    """Get a document by path or document ID."""
+    index_path = ctx.obj["index_path"]
+
+    if not index_path.exists():
+        console.print("[yellow]No index found.[/yellow]")
+        return
+
+    db = Database(index_path)
+    db.init_schema()
+
+    with db.connection:
+        doc_repo = DocumentRepository(db.connection)
+
+        # Try to find by ID first
+        doc = doc_repo.get_by_id(path_or_docid)
+
+        if not doc:
+            # Try to find by path
+            # Need collection ID - search all collections
+            from sif.database.repositories import CollectionRepository  # noqa: PLC0415
+
+            coll_repo = CollectionRepository(db.connection)
+
+            for coll in coll_repo.list_all():
+                doc = doc_repo.get_by_path(path_or_docid, coll.id)
+                if doc:
+                    break
+
+        if not doc:
+            # Try matching by filename (short name lookup)
+            doc = doc_repo.get_by_filename(path_or_docid)
+
+        if not doc:
+            # Try as file path
+            path = Path(path_or_docid)
+            if path.exists():
+                try:
+                    content = path.read_text()
+                    console.print(f"[bold]{path}[/bold]")
+                    console.print(content)
+                    return
+                except Exception as e:
+                    raise click.ClickException(f"Cannot read file: {e}") from e
+
+            raise click.ClickException(f"Document not found: {path_or_docid}")
+
+        # Display document
+        console.print(f"[bold cyan]{doc.title or doc.filename}[/bold cyan]")
+        console.print(f"[dim]Path: {doc.path}[/dim]")
+        console.print(f"[dim]Collection: {doc.collection_id}[/dim]")
+        console.print("")
+
+        # Get content
+        content = doc.content
+
+        # Apply line filtering
+        if from_line is not None or lines is not None:
+            content_lines = content.split("\n")
+
+            start = (from_line or 1) - 1  # Convert to 0-indexed
+            start = max(0, start)
+
+            if lines is not None:
+                end = start + lines
+                content_lines = content_lines[start:end]
+            else:
+                content_lines = content_lines[start:]
+
+            content = "\n".join(content_lines)
+
+        if line_numbers:
+            content = prepend_line_numbers(content)
+
+        # Display with syntax highlighting if markdown
+        if doc.path.endswith(".md"):
+            console.print(content)
+        else:
+            console.print(content)
+
+
+@get_group.command("multi-get")
+@click.argument("pattern")
+@click.option("--max-bytes", "-b", type=int, default=100000, help="Max bytes per file")
+@click.option("--line-numbers", is_flag=True, help="Show line numbers")
+@click.pass_context
+def multi_get_cmd(  # noqa: C901
+    ctx: click.Context,
+    pattern: str,
+    max_bytes: int,
+    line_numbers: bool,
+) -> None:
+    """Get multiple documents matching a pattern."""
+    index_path = ctx.obj["index_path"]
+
+    if not index_path.exists():
+        console.print("[yellow]No index found.[/yellow]")
+        return
+
+    import fnmatch  # noqa: PLC0415
+
+    db = Database(index_path)
+    db.init_schema()
+
+    with db.connection:
+        doc_repo = DocumentRepository(db.connection)
+        from sif.database.repositories import CollectionRepository  # noqa: PLC0415
+
+        coll_repo = CollectionRepository(db.connection)
+
+        matched_docs: list[Document] = []
+
+        def _lookup_by_id_or_path_or_filename(item: str) -> Document | None:
+            """Try ID, then exact path, then filename match."""
+            doc = doc_repo.get_by_id(item)
+            if doc:
+                return doc
+            for coll in coll_repo.list_all():
+                doc = doc_repo.get_by_path(item, coll.id)
+                if doc:
+                    return doc
+            return doc_repo.get_by_filename(item)
+
+        if "," in pattern:
+            # Comma-separated IDs/paths
+            items = [item.strip() for item in pattern.split(",") if item.strip()]
+            seen_ids: set[str] = set()
+            for item in items:
+                doc = _lookup_by_id_or_path_or_filename(item)
+                if doc and doc.id not in seen_ids:
+                    matched_docs.append(doc)
+                    seen_ids.add(doc.id)
+        elif "*" in pattern or "?" in pattern:
+            # Glob pattern
+            for coll in coll_repo.list_all():
+                matched_docs.extend(
+                    doc
+                    for doc in doc_repo.list_by_collection(coll.id)
+                    if fnmatch.fnmatch(doc.path, pattern) or fnmatch.fnmatch(doc.filename, pattern)
+                )
+        else:
+            # Single item
+            doc = _lookup_by_id_or_path_or_filename(pattern)
+            if doc:
+                matched_docs.append(doc)
+
+        if not matched_docs:
+            console.print(f"[yellow]No documents matching pattern: {pattern}[/yellow]")
+            return
+
+        console.print(f"[green]Found {len(matched_docs)} matching document(s)[/green]")
+        console.print("")
+
+        for doc in matched_docs:
+            content = doc.content
+            if len(content.encode()) > max_bytes:
+                content = content[:max_bytes] + "\n... [truncated]"
+
+            if line_numbers:
+                content = prepend_line_numbers(content)
+
+            console.print(f"[bold cyan]=== {doc.path} ===[/bold cyan]")
+            console.print(content)
+            console.print("")

sif/cli/commands/index.py

@@ -0,0 +1,294 @@
+"""Index management commands."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn
+
+from sif.core.models import Document
+from sif.database.database import Database
+from sif.database.repositories import (
+    CollectionRepository,
+    DocumentChunkRepository,
+    DocumentRepository,
+)
+from sif.indexing.chunker import create_chunker
+from sif.indexing.parser import MarkdownParser
+from sif.indexing.scanner import FileScanner
+
+
+console = Console()
+
+
+@click.group("index")
+def index_group() -> None:
+    """Index management commands."""
+    pass
+
+
+@index_group.command("update")
+@click.option("--collection", "-c", help="Update specific collection only")
+@click.option("--force", "-f", is_flag=True, help="Force re-index all documents")
+@click.pass_context
+def update_cmd(ctx: click.Context, collection: str | None, force: bool) -> None:  # noqa: C901, PLR0912, PLR0915
+    """Update the index by scanning collections."""
+    index_path = ctx.obj["index_path"]
+
+    db = Database(index_path)
+    db.init_schema()
+
+    with db.connection:
+        coll_repo = CollectionRepository(db.connection)
+        doc_repo = DocumentRepository(db.connection)
+
+        # Get collections to update
+        if collection:
+            collections = [coll_repo.get_by_name(collection)]
+            if not collections[0]:
+                raise click.ClickException(f"Collection '{collection}' not found")
+        else:
+            collections = coll_repo.list_all()
+
+        if not collections:
+            console.print(
+                "[yellow]No collections found. Add one with 'sif collection add'.[/yellow]"
+            )
+            return
+
+        total_added = 0
+        total_updated = 0
+        total_removed = 0
+
+        for coll in collections:
+            if not coll:
+                continue
+
+            console.print(f"\n[bold]Updating collection: {coll.name}[/bold]")
+
+            if coll.pre_update_cmd:
+                console.print(f"  Running pre-update command: {coll.pre_update_cmd}")
+                result = subprocess.run(
+                    coll.pre_update_cmd,
+                    shell=True,
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                )
+                if result.returncode != 0:
+                    err_msg = (
+                        f"Pre-update command failed for '{coll.name}' "
+                        f"(exit {result.returncode}): "
+                        f"{result.stderr.strip() or result.stdout.strip()}"
+                    )
+                    raise click.ClickException(err_msg)
+
+            # Scan files
+            scanner = FileScanner()
+            scan_result = scanner.scan(
+                Path(coll.path),
+                pattern=coll.pattern,
+                ignore_patterns=coll.ignore_patterns,
+            )
+
+            console.print(f"  Found {scan_result.file_count} files")
+
+            # Get existing documents
+            existing_docs = {d.path: d for d in doc_repo.list_by_collection(coll.id)}
+            scanned_paths = set()
+
+            # Process each file
+            parser = MarkdownParser()
+
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                task = progress.add_task(f"Indexing {coll.name}...", total=scan_result.file_count)
+
+                for file_path in scan_result.files:
+                    scanned_paths.add(str(file_path))
+
+                    # Parse document
+                    try:
+                        parsed = parser.parse(file_path)
+                    except Exception as e:
+                        console.print(f"  [red]Error parsing {file_path}: {e}[/red]")
+                        progress.advance(task)
+                        continue
+
+                    # Check if document exists
+                    existing = existing_docs.get(str(file_path))
+
+                    if existing:
+                        # Check if changed
+                        if existing.checksum == parsed.checksum and not force:
+                            console.print(f"  [dim]Unchanged: {file_path.name}[/dim]")
+                            progress.advance(task)
+                            continue
+
+                        # Update document
+                        existing.content = parsed.content
+                        existing.title = parsed.title
+                        existing.metadata = parsed.metadata
+                        existing.mtime = parsed.mtime
+                        doc_repo.update(existing)
+                        total_updated += 1
+                    else:
+                        # Create new document
+                        doc = Document(
+                            path=str(file_path),
+                            collection_id=coll.id,
+                            content=parsed.content,
+                            title=parsed.title,
+                            metadata=parsed.metadata,
+                            mtime=parsed.mtime,
+                        )
+                        doc_repo.create(doc)
+                        total_added += 1
+
+                    progress.advance(task)
+
+            # Remove documents that no longer exist
+            for path, doc in existing_docs.items():
+                if path not in scanned_paths:
+                    doc_repo.delete(doc.id)
+                    total_removed += 1
+
+            # Update collection stats
+            coll.document_count = len(doc_repo.list_by_collection(coll.id))
+            from datetime import datetime, timezone  # noqa: PLC0415
+
+            coll.last_indexed_at = datetime.now(timezone.utc)
+            coll_repo.update(coll)
+
+        console.print("\n[green]Index updated:[/green]")
+        console.print(f"  Added: {total_added}")
+        console.print(f"  Updated: {total_updated}")
+        console.print(f"  Removed: {total_removed}")
+
+
+@index_group.command("embed")
+@click.option("--collection", "-c", help="Embed specific collection only")
+@click.option("--force", "-f", is_flag=True, help="Force re-embed all documents")
+@click.option("--chunk-strategy", default="auto", help="Chunking strategy")
+@click.option("--model", "-m", help="Embedding model name")
+@click.option(
+    "--model-type",
+    type=click.Choice(["sentence_transformers", "gguf", "openai", "modelscope"]),
+    help="Embedding model type override",
+)
+@click.pass_context
+def embed_cmd(  # noqa: C901, PLR0912, PLR0913, PLR0915
+    ctx: click.Context,
+    collection: str | None,
+    force: bool,  # noqa: ARG001
+    chunk_strategy: str,
+    model: str | None,
+    model_type: str | None = None,
+) -> None:
+    """Generate embeddings for documents."""
+    index_path = ctx.obj["index_path"]
+
+    db = Database(index_path)
+    db.init_schema()
+
+    # Initialize embedder
+    from sif.config.settings import get_settings  # noqa: PLC0415
+    from sif.embedding.manager import EmbeddingManager  # noqa: PLC0415
+    from sif.search.vector import VectorSearcher  # noqa: PLC0415
+
+    settings = get_settings()
+    if model:
+        settings = settings.model_copy(update={"model_name": model})
+    if model_type:
+        settings = settings.model_copy(update={"model_type": model_type})
+
+    try:
+        manager = EmbeddingManager.from_settings(settings)
+    except ImportError as e:
+        console.print(f"[red]Embedding backend not installed: {e}[/red]")
+        return
+    except Exception as e:
+        console.print(f"[red]Failed to load embedding model: {e}[/red]")
+        return
+
+    with db.connection:
+        coll_repo = CollectionRepository(db.connection)
+        doc_repo = DocumentRepository(db.connection)
+        chunk_repo = DocumentChunkRepository(db.connection)
+
+        # Get collections
+        if collection:
+            collections = [coll_repo.get_by_name(collection)]
+            if not collections[0]:
+                raise click.ClickException(f"Collection '{collection}' not found")
+        else:
+            collections = coll_repo.list_all()
+
+        total_chunks = 0
+
+        for coll in collections:
+            if not coll:
+                continue
+
+            console.print(f"\n[bold]Embedding collection: {coll.name}[/bold]")
+            documents = doc_repo.list_by_collection(coll.id)
+            chunker = create_chunker(chunk_strategy)
+
+            # Collect all chunks across documents
+            all_chunks = []  # (chunk, document_id)
+            doc_chunks_map: dict[str, list] = {}
+
+            for doc in documents:
+                chunk_repo.delete_by_document(doc.id)
+                chunks = chunker.chunk(doc.content)
+                doc_chunks_map[doc.id] = chunks
+                for i, chunk in enumerate(chunks):
+                    chunk.document_id = doc.id
+                    chunk.sequence = i
+                    all_chunks.append((chunk, doc.id))
+
+            # Batch embed all chunks
+            if all_chunks:
+                try:
+                    chunk_texts = [c.content for c, _ in all_chunks]
+                    embedding_response = manager.embed(chunk_texts)
+                    embeddings = embedding_response.embeddings
+
+                    # Persist chunks and embeddings
+                    batch_items: list[tuple[str, str, str | None, list[float]]] = []
+                    for (chunk, doc_id), embedding in zip(all_chunks, embeddings):
+                        chunk_repo.create(chunk)
+                        batch_items.append((chunk.id, doc_id, chunk.id, embedding))
+
+                    if batch_items:
+                        vector_searcher = VectorSearcher(
+                            db.connection, len(manager.embed_single("probe"))
+                        )
+                        vector_searcher.add_embeddings_batch(batch_items)
+
+                    total_chunks += len(all_chunks)
+                except Exception as e:
+                    console.print(f"  [red]Error embedding collection {coll.name}: {e}[/red]")
+
+            # Update collection stats
+            documents = doc_repo.list_by_collection(coll.id)
+            coll.chunk_count = sum(len(chunk_repo.get_by_document(d.id)) for d in documents)
+            coll_repo.update(coll)
+
+        console.print(f"\n[green]Embedding complete: {total_chunks} chunks embedded[/green]")
+
+
+@index_group.command("status")
+@click.pass_context
+def index_status_cmd(ctx: click.Context) -> None:
+    """Show index status."""
+    # Delegate to main status command
+    from sif.cli.main import status_cmd  # noqa: PLC0415
+
+    ctx.invoke(status_cmd)

sif/cli/commands/ls.py

@@ -0,0 +1,67 @@
+"""Top-level ls command for indexed documents."""
+
+from __future__ import annotations
+
+import click
+from rich.console import Console
+from rich.tree import Tree
+
+from sif.database.database import Database
+from sif.database.repositories import CollectionRepository, DocumentRepository
+
+
+console = Console()
+
+
+@click.command("ls")
+@click.argument("collection", required=False)
+@click.argument("subpath", required=False)
+@click.pass_context
+def ls_cmd(
+    ctx: click.Context,
+    collection: str | None,
+    subpath: str | None,
+) -> None:
+    """List indexed documents as a virtual file tree."""
+    index_path = ctx.obj["index_path"]
+
+    if not index_path.exists():
+        console.print("[yellow]No index found.[/yellow]")
+        return
+
+    db = Database(index_path)
+    db.init_schema()
+
+    with db.connection:
+        doc_repo = DocumentRepository(db.connection)
+        coll_repo = CollectionRepository(db.connection)
+
+        if collection:
+            collections = [coll_repo.get_by_name(collection)]
+            if not collections[0]:
+                raise click.ClickException(f"Collection '{collection}' not found")
+        else:
+            collections = coll_repo.list_all()
+
+        for coll in collections:
+            if not coll:
+                continue
+
+            docs = doc_repo.list_by_collection(coll.id)
+            if subpath:
+                prefix = subpath.strip("/")
+                docs = [d for d in docs if d.path.strip("/").startswith(prefix)]
+
+            if not docs:
+                continue
+
+            tree = Tree(f"[bold cyan]{coll.name}[/bold cyan]")
+            for doc in sorted(docs, key=lambda d: d.path):
+                parts = doc.path.strip("/").split("/")
+                node = tree
+                for part in parts[:-1]:
+                    found = next((c for c in node.children if str(c.label) == part), None)
+                    node = found if found else node.add(part)
+                node.add(parts[-1])
+
+            console.print(tree)

sif/cli/commands/mcp.py

@@ -0,0 +1,93 @@
+"""MCP server commands."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import click
+from rich.console import Console
+
+
+console = Console()
+
+
+@click.group("mcp")
+def mcp_group() -> None:
+    """MCP server commands."""
+    pass
+
+
+@mcp_group.command("stdio")
+@click.pass_context
+def mcp_stdio_cmd(ctx: click.Context) -> None:
+    """Run MCP server in stdio mode."""
+    index_path = ctx.obj["index_path"]
+
+    if not Path(index_path).exists():
+        console.print(
+            "[yellow]Warning: No index found. Some features may not work.[/yellow]", file=sys.stderr
+        )
+
+    try:
+        from sif.mcp.server import run_stdio_server  # noqa: PLC0415
+
+        run_stdio_server(str(index_path))
+    except ImportError as e:
+        console.print(f"[red]MCP server not available: {e}[/red]")
+        raise click.ClickException("Install with: pip install sif[mcp]") from None
+
+
+@mcp_group.command("http")
+@click.option("--host", "-h", default="127.0.0.1", help="Host to bind to")
+@click.option("--port", "-p", type=int, default=3000, help="Port to listen on")
+@click.option("--reload", is_flag=True, help="Enable auto-reload")
+@click.pass_context
+def mcp_http_cmd(
+    ctx: click.Context,
+    host: str,
+    port: int,
+    reload: bool,
+) -> None:
+    """Run MCP server in HTTP mode."""
+    index_path = ctx.obj["index_path"]
+
+    console.print(f"[green]Starting MCP HTTP server on {host}:{port}[/green]")
+
+    try:
+        from sif.mcp.server_http import run_http_server  # noqa: PLC0415
+
+        run_http_server(str(index_path), host=host, port=port, reload=reload)
+    except ImportError as e:
+        console.print(f"[red]HTTP server not available: {e}[/red]")
+        raise click.ClickException("Install with: pip install sif[http]") from None
+
+
+@mcp_group.command("daemon")
+@click.option("--host", "-h", default="127.0.0.1", help="Host to bind to")
+@click.option("--port", "-p", type=int, default=3000, help="Port to listen on")
+@click.option("--pid-file", help="PID file path")
+@click.option("--log-file", help="Log file path")
+@click.option("--stop", is_flag=True, help="Stop the daemon")
+@click.pass_context
+def mcp_daemon_cmd(  # noqa: PLR0913
+    ctx: click.Context,
+    host: str,
+    port: int,
+    pid_file: str | None,  # noqa: ARG001
+    log_file: str | None,  # noqa: ARG001
+    stop: bool,
+) -> None:
+    """Run MCP server as a daemon."""
+    if stop:
+        console.print("[yellow]Stopping daemon...[/yellow]")
+        # Implementation would read PID file and kill process
+        console.print("[green]Daemon stopped[/green]")
+        return
+
+    console.print("[green]Starting MCP daemon...[/green]")
+    console.print(f"[dim]Host: {host}:{port}[/dim]")
+
+    # In a real implementation, this would daemonize the process
+    # For now, just run the HTTP server
+    ctx.invoke(mcp_http_cmd, host=host, port=port, reload=False)