wikigen 1.0.0__py3-none-any.whl

wikigen/mcp/server.py

@@ -0,0 +1,232 @@
+"""MCP server implementation for wikigen."""
+
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from mcp.server.fastmcp import FastMCP
+
+from .output_resources import discover_all_projects
+from ..config import get_output_dir
+from .search_index import FileIndexer
+
+# Initialize the MCP server
+# Instructions help editors/clients understand what this server provides
+app = FastMCP(
+    "wikigen",
+    instructions=(
+        "Expose local wiki markdown files as MCP tools. "
+        "Available tools: search_docs (semantic search across indexed directories), "
+        "get_docs (fetch content by resource name or file path), and index_directories (index dirs for search). "
+        "Doc names mirror paths under your configured output_dir (without .md extension)."
+    ),
+)
+
+# Initialize search indexer for fast machine-wide search
+_indexer: Optional[FileIndexer] = None
+
+
+def _get_indexer() -> FileIndexer:
+    """Get or create the search indexer instance."""
+    global _indexer
+    if _indexer is None:
+        _indexer = FileIndexer()
+    return _indexer
+
+
+# Store discovered projects (refreshed on each request)
+_projects: Dict[str, Path] = {}
+
+
+def _refresh_projects():
+    """Refresh the project registry."""
+    global _projects
+    _projects = discover_all_projects()
+
+
+def _get_project_resources():
+    """Get list of available project resources."""
+    _refresh_projects()
+    return _projects
+
+
+# MCP Tools - executable actions for interacting with documentation
+@app.tool()
+def get_docs(identifier: str) -> str:
+    """
+    Get the full content of a documentation file by resource name or absolute file path.
+
+    This tool can fetch documentation using either:
+    - Resource name (e.g., 'README', 'Order Management/Felis Stream') - searches output_dir
+    - Absolute file path (e.g., '/Users/name/Documents/doc.md') - works with any indexed file
+
+    Args:
+        identifier: Either a resource name (from output_dir) or an absolute file path
+
+    Returns:
+        The markdown content of the requested documentation file.
+    """
+    # Check if it looks like an absolute path
+    path_obj = Path(identifier)
+    if path_obj.is_absolute() and path_obj.exists():
+        # Treat as absolute file path
+        try:
+            content = path_obj.read_text(encoding="utf-8")
+            return content
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to read file at path '{identifier}': {e}"
+            ) from e
+
+    # Treat as resource name from output_dir
+    projects = _get_project_resources()
+
+    if identifier not in projects:
+        # Provide helpful error message
+        available = ", ".join(sorted(projects.keys())[:10]) if projects else "none"
+        if len(projects) > 10:
+            available += f" ... (and {len(projects) - 10} more)"
+        raise ValueError(
+            f"Documentation '{identifier}' not found in output directory. "
+            f"Available resources: {available}. "
+            f"If you meant to use a file path, provide an absolute path starting with '/'."
+        )
+
+    doc_path = projects[identifier]
+    try:
+        content = doc_path.read_text(encoding="utf-8")
+        return content
+    except Exception as e:
+        raise RuntimeError(
+            f"Failed to read documentation for '{identifier}': {e}"
+        ) from e
+
+
+@app.tool()
+def search_docs(
+    query: str,
+    limit: int = 20,
+    directory_filter: Optional[str] = None,
+    chunk_limit: int = 5,
+) -> str:
+    """
+    Search for markdown files across indexed directories using semantic search.
+
+    This tool uses semantic search to find relevant chunks from indexed documentation.
+    It returns relevant chunks with content snippets instead of entire files.
+    Index directories first using index_directories, or files are auto-indexed from
+    the configured output_dir on first search.
+
+    Args:
+        query: Search query (supports multi-word queries and natural language)
+        limit: Maximum number of chunks to return (default: 20)
+        directory_filter: Optional directory path to filter results
+        chunk_limit: Maximum chunks per file (default: 5)
+
+    Returns:
+        Formatted list of relevant chunks with their paths, resource names, scores, and content snippets.
+    """
+    indexer = _get_indexer()
+
+    # Auto-index default output directory if no files indexed yet
+    stats = indexer.get_stats()
+    if stats["total_files"] == 0:
+        output_dir = get_output_dir()
+        if output_dir.exists():
+            added, updated, skipped = indexer.index_directory(output_dir)
+            if added > 0 or updated > 0:
+                return f"Indexed {added} new files, updated {updated}. Try searching again."
+
+    # Always use semantic search
+    if not indexer.enable_semantic_search or not indexer.vector_index:
+        # Fallback to keyword search if semantic search is not available
+        results = indexer.search(query, limit=limit, directory_filter=directory_filter)
+        if not results:
+            return f"No files found matching '{query}'."
+
+        # Format file results
+        lines = [f"Found {len(results)} file(s) matching '{query}':\n"]
+        for i, result in enumerate(results, 1):
+            lines.append(
+                f"{i}. {result['resource_name']}\n"
+                f"   Path: {result['file_path']}\n"
+                f"   Directory: {result['directory']}"
+            )
+        return "\n".join(lines)
+
+    # Use semantic search
+    results = indexer.search_semantic(
+        query,
+        limit=limit,
+        directory_filter=directory_filter,
+        max_chunks_per_file=chunk_limit,
+    )
+
+    if not results:
+        return f"No chunks found matching '{query}'."
+
+    # Format chunk results
+    lines = [f"Found {len(results)} relevant chunk(s) matching '{query}':\n"]
+    for i, result in enumerate(results, 1):
+        # Truncate chunk content for display (first 200 chars)
+        content_snippet = result.get("content", "")[:200]
+        if len(result.get("content", "")) > 200:
+            content_snippet += "..."
+
+        lines.append(
+            f"{i}. {result['resource_name']} (chunk {result.get('chunk_index', 0)})\n"
+            f"   Path: {result['file_path']}\n"
+            f"   Score: {result.get('score', 0):.4f}\n"
+            f"   Content: {content_snippet}"
+        )
+
+    return "\n".join(lines)
+
+
+@app.tool()
+def index_directories(directories: List[str], max_depth: Optional[int] = None) -> str:
+    """
+    Index markdown files from specified directories for fast searching.
+
+    Args:
+        directories: List of directory paths to index
+        max_depth: Maximum recursion depth (None = unlimited)
+
+    Returns:
+        Summary of indexing results.
+    """
+    indexer = _get_indexer()
+
+    total_added = 0
+    total_updated = 0
+    total_skipped = 0
+
+    results = []
+
+    for dir_path in directories:
+        path = Path(dir_path).expanduser()
+        if not path.exists():
+            results.append(f"✗ {dir_path}: Directory does not exist")
+            continue
+
+        if not path.is_dir():
+            results.append(f"✗ {dir_path}: Path is not a directory")
+            continue
+
+        added, updated, skipped = indexer.index_directory(path, max_depth=max_depth)
+        total_added += added
+        total_updated += updated
+        total_skipped += skipped
+
+        results.append(
+            f"✓ {dir_path}: {added} added, {updated} updated, {skipped} skipped"
+        )
+
+    summary = "\n".join(results)
+    summary += f"\n\nTotal: {total_added} added, {total_updated} updated, {total_skipped} skipped"
+
+    return summary
+
+
+def run_mcp_server():
+    """Entry point to run MCP server."""
+    app.run()

wikigen/mcp/vector_index.py

@@ -0,0 +1,297 @@
+"""FAISS vector index management for semantic search.
+
+This module provides FAISS index management for storing and searching
+document chunk embeddings.
+"""
+
+import pickle
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+from threading import Lock
+import numpy as np
+
+try:
+    import faiss
+
+    FAISS_AVAILABLE = True
+except ImportError:
+    FAISS_AVAILABLE = False
+    faiss = None
+
+from ..config import CONFIG_DIR
+
+
+class VectorIndex:
+    """
+    FAISS vector index manager for semantic search.
+
+    Manages a FAISS index for storing and searching document chunk embeddings.
+    Also maintains metadata mapping chunk IDs to file paths and chunk information.
+    """
+
+    def __init__(self, index_path: Optional[Path] = None, embedding_dim: int = 384):
+        """
+        Initialize the vector index.
+
+        Args:
+            index_path: Path to save/load the FAISS index. Defaults to config_dir/vector_index.faiss
+            embedding_dim: Dimension of embeddings (default: 384 for all-MiniLM-L6-v2)
+        """
+        if not FAISS_AVAILABLE:
+            raise ImportError(
+                "FAISS is not available. Please install faiss-cpu: pip install faiss-cpu"
+            )
+
+        if index_path is None:
+            index_path = CONFIG_DIR / "vector_index.faiss"
+
+        self.index_path = index_path
+        self.metadata_path = index_path.with_suffix(".metadata.pkl")
+        self.embedding_dim = embedding_dim
+        self._lock = Lock()
+
+        # FAISS index (FlatIndex for exact search)
+        self.index: Optional[faiss.Index] = None
+
+        # Metadata: chunk_id -> {file_path, chunk_index, content, start_pos, end_pos}
+        self.metadata: Dict[int, Dict[str, Any]] = {}
+
+        # File to chunk IDs mapping: file_path -> [chunk_id, ...]
+        self.file_to_chunks: Dict[str, List[int]] = {}
+
+        # Next chunk ID
+        self.next_chunk_id = 0
+
+        # Load existing index if available
+        self._load()
+
+    def _load(self) -> None:
+        """Load FAISS index and metadata from disk."""
+        with self._lock:
+            if self.index_path.exists() and self.metadata_path.exists():
+                try:
+                    # Load FAISS index
+                    self.index = faiss.read_index(str(self.index_path))
+
+                    # Load metadata
+                    with open(self.metadata_path, "rb") as f:
+                        data = pickle.load(f)
+                        self.metadata = data.get("metadata", {})
+                        self.file_to_chunks = data.get("file_to_chunks", {})
+                        self.next_chunk_id = data.get("next_chunk_id", 0)
+
+                    # Verify embedding dimension matches
+                    if self.index.d != self.embedding_dim:
+                        raise ValueError(
+                            f"Index embedding dimension ({self.index.d}) "
+                            f"does not match expected ({self.embedding_dim})"
+                        )
+                except Exception as e:
+                    # If loading fails, start fresh
+                    print(f"Warning: Could not load vector index: {e}")
+                    self._init_index()
+            else:
+                self._init_index()
+
+    def _init_index(self) -> None:
+        """Initialize a new FAISS index."""
+        # Use FlatIndex for exact search (good for <1M vectors)
+        # For larger datasets, consider IVF or HNSW
+        self.index = faiss.IndexFlatL2(self.embedding_dim)
+        self.metadata = {}
+        self.file_to_chunks = {}
+        self.next_chunk_id = 0
+
+    def add_chunks(
+        self,
+        file_path: str,
+        chunks: List[Dict[str, Any]],
+        embeddings: np.ndarray,
+    ) -> None:
+        """
+        Add chunks and their embeddings to the index.
+
+        Args:
+            file_path: Path to the file these chunks belong to
+            chunks: List of chunk dictionaries with 'content', 'start_pos', 'end_pos', 'chunk_index'
+            embeddings: NumPy array of embeddings (shape: (len(chunks), embedding_dim))
+        """
+        if not FAISS_AVAILABLE:
+            raise ImportError("FAISS is not available")
+
+        if len(chunks) != len(embeddings):
+            raise ValueError(
+                f"Number of chunks ({len(chunks)}) does not match "
+                f"number of embeddings ({len(embeddings)})"
+            )
+
+        with self._lock:
+            # Remove existing chunks for this file
+            if file_path in self.file_to_chunks:
+                self._remove_file(file_path)
+
+            # Ensure embeddings are float32 and 2D
+            if embeddings.dtype != np.float32:
+                embeddings = embeddings.astype(np.float32)
+            if len(embeddings.shape) == 1:
+                embeddings = embeddings.reshape(1, -1)
+
+            # Add embeddings to FAISS index
+            self.index.add(embeddings)
+
+            # Add metadata for each chunk
+            chunk_ids = []
+            for i, chunk in enumerate(chunks):
+                chunk_id = self.next_chunk_id
+                chunk_ids.append(chunk_id)
+
+                self.metadata[chunk_id] = {
+                    "file_path": file_path,
+                    "chunk_index": chunk.get("chunk_index", i),
+                    "content": chunk.get("content", ""),
+                    "start_pos": chunk.get("start_pos", 0),
+                    "end_pos": chunk.get("end_pos", 0),
+                }
+
+                self.next_chunk_id += 1
+
+            # Update file to chunks mapping
+            self.file_to_chunks[file_path] = chunk_ids
+
+    def search(
+        self,
+        query_embedding: np.ndarray,
+        k: int = 10,
+        file_filter: Optional[List[str]] = None,
+    ) -> List[Tuple[int, float, Dict[str, Any]]]:
+        """
+        Search for similar chunks.
+
+        Args:
+            query_embedding: Query embedding vector
+            k: Number of results to return
+            file_filter: Optional list of file paths to filter results
+
+        Returns:
+            List of tuples: (chunk_id, distance, metadata_dict)
+            Sorted by distance (lower is better)
+        """
+        if not FAISS_AVAILABLE:
+            raise ImportError("FAISS is not available")
+
+        if self.index is None or self.index.ntotal == 0:
+            return []
+
+        with self._lock:
+            # Ensure query embedding is float32 and 2D
+            if query_embedding.dtype != np.float32:
+                query_embedding = query_embedding.astype(np.float32)
+            if len(query_embedding.shape) == 1:
+                query_embedding = query_embedding.reshape(1, -1)
+
+            # Search in FAISS
+            distances, indices = self.index.search(
+                query_embedding, k * 2
+            )  # Get more, filter later
+
+            # Filter and format results
+            results = []
+            seen_files = {}  # Track chunks per file for file_filter
+
+            for idx, dist in zip(indices[0], distances[0]):
+                if idx < 0:  # Invalid index
+                    continue
+
+                chunk_id = idx
+                if chunk_id not in self.metadata:
+                    continue
+
+                metadata = self.metadata[chunk_id]
+                file_path = metadata["file_path"]
+
+                # Apply file filter if provided
+                if file_filter is not None and file_path not in file_filter:
+                    continue
+
+                # Limit chunks per file (if file_filter is used)
+                if file_filter is not None:
+                    if file_path not in seen_files:
+                        seen_files[file_path] = 0
+                    if seen_files[file_path] >= 5:  # Max chunks per file
+                        continue
+                    seen_files[file_path] += 1
+
+                results.append((chunk_id, float(dist), metadata))
+
+                if len(results) >= k:
+                    break
+
+            return results
+
+    def _remove_file(self, file_path: str) -> None:
+        """Remove all chunks for a file (internal method, not thread-safe)."""
+        if file_path not in self.file_to_chunks:
+            return
+
+        chunk_ids = self.file_to_chunks[file_path]
+
+        # Note: FAISS doesn't support removing individual vectors efficiently
+        # For now, we'll mark them as removed in metadata and rebuild on next save
+        # A better approach would be to rebuild the index, but that's expensive
+        # For production, consider using a more advanced index type that supports deletion
+
+        # Remove from metadata
+        for chunk_id in chunk_ids:
+            self.metadata.pop(chunk_id, None)
+
+        # Remove from file mapping
+        del self.file_to_chunks[file_path]
+
+    def remove_file(self, file_path: str) -> None:
+        """
+        Remove all chunks for a file from the index.
+
+        Note: This marks chunks as removed but doesn't actually remove them
+        from the FAISS index. The index will be rebuilt on next save.
+
+        Args:
+            file_path: Path to the file to remove
+        """
+        with self._lock:
+            self._remove_file(file_path)
+
+    def save(self) -> None:
+        """Save FAISS index and metadata to disk."""
+        if not FAISS_AVAILABLE:
+            return
+
+        with self._lock:
+            if self.index is None:
+                return
+
+            # Ensure directory exists
+            self.index_path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Save FAISS index
+            faiss.write_index(self.index, str(self.index_path))
+
+            # Save metadata
+            with open(self.metadata_path, "wb") as f:
+                pickle.dump(
+                    {
+                        "metadata": self.metadata,
+                        "file_to_chunks": self.file_to_chunks,
+                        "next_chunk_id": self.next_chunk_id,
+                    },
+                    f,
+                )
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get statistics about the index."""
+        with self._lock:
+            return {
+                "total_chunks": len(self.metadata),
+                "total_files_with_chunks": len(self.file_to_chunks),
+                "index_size": self.index.ntotal if self.index else 0,
+                "embedding_dim": self.embedding_dim,
+            }

wikigen/metadata/__init__.py

@@ -0,0 +1,35 @@
+"""
+Metadata package for WikiGen.
+Centralized source of truth for project information.
+"""
+
+from .project import (
+    PROJECT_NAME,
+    AUTHOR_NAME,
+    ORGANIZATION,
+    DESCRIPTION,
+    REPOSITORY_URL,
+    HOMEPAGE_URL,
+    ISSUES_URL,
+    COPYRIGHT_TEXT,
+    MIN_PYTHON_VERSION,
+    CLI_ENTRY_POINT,
+)
+
+from .version import get_version, __version__
+
+# Re-export commonly used items
+__all__ = [
+    "PROJECT_NAME",
+    "AUTHOR_NAME",
+    "ORGANIZATION",
+    "DESCRIPTION",
+    "REPOSITORY_URL",
+    "HOMEPAGE_URL",
+    "ISSUES_URL",
+    "COPYRIGHT_TEXT",
+    "MIN_PYTHON_VERSION",
+    "CLI_ENTRY_POINT",
+    "get_version",
+    "__version__",
+]

wikigen/metadata/logo.py

@@ -0,0 +1,28 @@
+"""ASCII logo for WikiGen CLI."""
+
+from .project import DESCRIPTION
+from .version import get_version
+
+
+def print_logo():
+    """Print the WikiGen ASCII logo with simple gray colors."""
+    # Simple colors that work well on both light and dark backgrounds
+    LOGO_COLOR = "\033[38;5;240m"  # Medium gray - visible everywhere
+    ATTRIB_COLOR = "\033[38;5;245m"  # Light gray
+    RESET = "\033[0m"
+
+    version = get_version()
+    logo = f"""
+{ATTRIB_COLOR}INTRODUCING
+{RESET}{LOGO_COLOR}
+██╗   ██╗      ██╗    ██╗ ██╗ ██╗  ██╗ ██╗  ██████╗  ███████╗ ███╗   ██╗
+  ██║   ██║    ██║    ██║ ██║ ██║ ██╔╝ ██║ ██╔════╝  ██╔════╝ ████╗  ██║
+    ██║   ██║  ██║ █╗ ██║ ██║ █████╔╝  ██║ ██║  ███╗ ██████╗  ██╔██╗ ██║
+  ██║   ██║    ██║███╗██║ ██║ ██╔═██╗  ██║ ██║   ██║ ██╔══╝   ██║╚██╗██║
+██║   ██║      ╚███╔███╔╝ ██║ ██║  ██╗ ██║ ╚██████╔╝ ███████╗ ██║ ╚████║
+╚═╝   ╚═╝       ╚══╝╚══╝  ╚═╝ ╚═╝  ╚═╝ ╚═╝  ╚═════╝  ╚══════╝ ╚═╝  ╚═══╝
+{RESET}
+{ATTRIB_COLOR}{DESCRIPTION} ♥ {RESET}
+{ATTRIB_COLOR}v{version}{RESET}
+"""
+    print(logo)

wikigen/metadata/project.py

@@ -0,0 +1,28 @@
+"""
+Project metadata for WikiGen.
+Single source of truth for project information.
+"""
+
+import datetime
+
+# Project info
+PROJECT_NAME = "wikigen"
+AUTHOR_NAME = "Mithun Ramesh"
+ORGANIZATION = "USEWIKIGEN.CO"
+DESCRIPTION = "WIKI'S FOR NERDS, BY NERDS"
+
+# Repository info
+REPOSITORY_URL = "https://github.com/usesalt/wikigen"
+HOMEPAGE_URL = "https://usesalt.co"
+ISSUES_URL = "https://github.com/usesalt/wikigen/issues"
+
+# Dynamic values
+CURRENT_YEAR = datetime.datetime.now().year
+COPYRIGHT_TEXT = f"Copyright (c) {CURRENT_YEAR} {AUTHOR_NAME}"
+
+# Python requirements
+MIN_PYTHON_VERSION = "3.12"
+
+# Package info
+PACKAGE_NAME = "wikigen"
+CLI_ENTRY_POINT = "wikigen"

wikigen/metadata/version.py

@@ -0,0 +1,17 @@
+"""
+Version management for WikiGen.
+Centralized version definition for consistency.
+"""
+
+# Current version - update this when releasing
+__version__ = "1.0.0"
+
+
+def get_version():
+    """
+    Get the current version.
+
+    Returns:
+        str: The current version string (e.g., "1.0.0")
+    """
+    return __version__

wikigen/nodes/__init__.py

@@ -0,0 +1 @@
+"""Nodes module for WikiGen."""