mcp-vector-search 0.15.7__py3-none-any.whl

mcp_vector_search/parsers/text.py

@@ -0,0 +1,186 @@
+"""Text file parser for MCP Vector Search."""
+
+from pathlib import Path
+
+from ..config.constants import TEXT_CHUNK_SIZE
+from ..core.models import CodeChunk
+from .base import BaseParser
+
+
+class TextParser(BaseParser):
+    """Parser for plain text and markdown files (.txt, .md, .markdown)."""
+
+    def __init__(self) -> None:
+        """Initialize text parser."""
+        super().__init__("text")
+
+    async def parse_file(self, file_path: Path) -> list[CodeChunk]:
+        """Parse a text file and extract chunks.
+
+        Args:
+            file_path: Path to the text file
+
+        Returns:
+            List of text chunks
+        """
+        try:
+            with open(file_path, encoding="utf-8") as f:
+                content = f.read()
+            return await self.parse_content(content, file_path)
+        except Exception:
+            # Return empty list if file can't be read
+            return []
+
+    async def parse_content(self, content: str, file_path: Path) -> list[CodeChunk]:
+        """Parse text content into semantic chunks.
+
+        Uses paragraph-based chunking for better semantic coherence.
+        Falls back to line-based chunking for non-paragraph text.
+
+        Args:
+            content: Text content to parse
+            file_path: Path to the source file
+
+        Returns:
+            List of text chunks
+        """
+        if not content.strip():
+            return []
+
+        chunks = []
+        lines = content.splitlines(keepends=True)
+
+        # Try paragraph-based chunking first
+        paragraphs = self._extract_paragraphs(content)
+
+        if paragraphs:
+            # Use paragraph-based chunking
+            for para_info in paragraphs:
+                chunk = self._create_chunk(
+                    content=para_info["content"],
+                    file_path=file_path,
+                    start_line=para_info["start_line"],
+                    end_line=para_info["end_line"],
+                    chunk_type="text",
+                )
+                chunks.append(chunk)
+        else:
+            # Fall back to line-based chunking for non-paragraph text
+            # Use smaller chunks for text files (30 lines instead of 50)
+            chunk_size = TEXT_CHUNK_SIZE
+            for i in range(0, len(lines), chunk_size):
+                start_line = i + 1
+                end_line = min(i + chunk_size, len(lines))
+
+                chunk_content = "".join(lines[i:end_line])
+
+                if chunk_content.strip():
+                    chunk = self._create_chunk(
+                        content=chunk_content,
+                        file_path=file_path,
+                        start_line=start_line,
+                        end_line=end_line,
+                        chunk_type="text",
+                    )
+                    chunks.append(chunk)
+
+        return chunks
+
+    def _extract_paragraphs(self, content: str) -> list[dict]:
+        """Extract paragraphs from text content.
+
+        A paragraph is defined as one or more non-empty lines
+        separated by empty lines.
+
+        Args:
+            content: Text content
+
+        Returns:
+            List of paragraph info dictionaries
+        """
+        lines = content.splitlines(keepends=True)
+        paragraphs = []
+        current_para = []
+        start_line = 1
+
+        for i, line in enumerate(lines, 1):
+            if line.strip():
+                if not current_para:
+                    start_line = i
+                current_para.append(line)
+            else:
+                if current_para:
+                    # End of paragraph
+                    para_content = "".join(current_para)
+                    if len(para_content.strip()) > 20:  # Minimum paragraph size
+                        paragraphs.append(
+                            {
+                                "content": para_content,
+                                "start_line": start_line,
+                                "end_line": i - 1,
+                            }
+                        )
+                    current_para = []
+
+        # Handle last paragraph if exists
+        if current_para:
+            para_content = "".join(current_para)
+            if len(para_content.strip()) > 20:
+                paragraphs.append(
+                    {
+                        "content": para_content,
+                        "start_line": start_line,
+                        "end_line": len(lines),
+                    }
+                )
+
+        # If we have very few paragraphs, merge small ones
+        if paragraphs:
+            merged = self._merge_small_paragraphs(paragraphs)
+            return merged
+
+        return []
+
+    def _merge_small_paragraphs(
+        self, paragraphs: list[dict], target_size: int = 200
+    ) -> list[dict]:
+        """Merge small paragraphs to create more substantial chunks.
+
+        Args:
+            paragraphs: List of paragraph dictionaries
+            target_size: Target size for merged paragraphs in characters
+
+        Returns:
+            List of merged paragraph dictionaries
+        """
+        merged = []
+        current_merge = None
+
+        for para in paragraphs:
+            para_len = len(para["content"])
+
+            if current_merge is None:
+                current_merge = para.copy()
+            elif len(current_merge["content"]) + para_len < target_size * 2:
+                # Merge with current
+                current_merge["content"] += "\n" + para["content"]
+                current_merge["end_line"] = para["end_line"]
+            else:
+                # Start new merge
+                if len(current_merge["content"].strip()) > 20:
+                    merged.append(current_merge)
+                current_merge = para.copy()
+
+        # Add last merge
+        if current_merge and len(current_merge["content"].strip()) > 20:
+            merged.append(current_merge)
+
+        return merged
+
+    def get_supported_extensions(self) -> list[str]:
+        """Get list of supported file extensions.
+
+        Returns:
+            List of supported extensions
+        """
+        return [".txt", ".md", ".markdown"]

mcp_vector_search/parsers/utils.py

@@ -0,0 +1,265 @@
+"""Shared utilities for language parsers.
+
+This module contains common functionality used across multiple parsers
+to reduce code duplication and improve maintainability.
+"""
+
+from pathlib import Path
+from re import Pattern
+
+from ..config.constants import DEFAULT_CHUNK_SIZE
+from ..core.models import CodeChunk
+
+
+def split_into_lines(content: str) -> list[str]:
+    """Split content into lines, handling different line endings.
+
+    Args:
+        content: Text content to split
+
+    Returns:
+        List of lines with line endings preserved
+    """
+    # Handle different line endings and preserve them
+    return content.splitlines(keepends=True)
+
+
+def get_line_range(lines: list[str], start_line: int, end_line: int) -> str:
+    """Get content from a range of lines.
+
+    Args:
+        lines: List of lines
+        start_line: Starting line number (1-indexed)
+        end_line: Ending line number (1-indexed, inclusive)
+
+    Returns:
+        Joined content from the line range
+    """
+    # Convert to 0-indexed
+    start_idx = max(0, start_line - 1)
+    end_idx = min(len(lines), end_line)
+
+    return "".join(lines[start_idx:end_idx])
+
+
+def find_block_end(lines: list[str], start_line: int, indent_char: str = " ") -> int:
+    """Find the end of a code block based on indentation.
+
+    This is a simple heuristic that looks for the next line with equal or
+    lower indentation level than the starting line.
+
+    Args:
+        lines: List of lines
+        start_line: Starting line number (1-indexed)
+        indent_char: Character used for indentation (space or tab)
+
+    Returns:
+        End line number (1-indexed)
+    """
+    if start_line > len(lines):
+        return len(lines)
+
+    # Get indentation of starting line
+    start_idx = start_line - 1
+    start_indent = len(lines[start_idx]) - len(lines[start_idx].lstrip())
+
+    # Find next line with same or lower indentation
+    for i in range(start_idx + 1, len(lines)):
+        line = lines[i]
+        if line.strip():  # Skip empty lines
+            current_indent = len(line) - len(line.lstrip())
+            if current_indent <= start_indent:
+                return i  # Return 0-indexed position, will be used as end_line
+
+    return len(lines)
+
+
+def create_simple_chunks(
+    content: str, file_path: Path, chunk_size: int = DEFAULT_CHUNK_SIZE
+) -> list[CodeChunk]:
+    """Create simple line-based chunks from content.
+
+    This is a fallback chunking strategy when more sophisticated
+    parsing is not available.
+
+    Args:
+        content: File content
+        file_path: Path to source file
+        chunk_size: Number of lines per chunk
+
+    Returns:
+        List of code chunks
+    """
+    lines = split_into_lines(content)
+    chunks = []
+
+    for i in range(0, len(lines), chunk_size):
+        start_line = i + 1
+        end_line = min(i + chunk_size, len(lines))
+
+        chunk_content = get_line_range(lines, start_line, end_line)
+
+        if chunk_content.strip():
+            chunk = CodeChunk(
+                content=chunk_content,
+                start_line=start_line,
+                end_line=end_line,
+                file_path=str(file_path),
+                chunk_type="block",
+                metadata={"source": "simple_chunking"},
+            )
+            chunks.append(chunk)
+
+    return chunks
+
+
+def extract_docstring(lines: list[str], start_line: int) -> str | None:
+    """Extract docstring/comment block starting from a given line.
+
+    Supports Python docstrings (triple quotes), JavaDoc (/** */),
+    and hash-based comments (# or //).
+
+    Args:
+        lines: List of lines
+        start_line: Line number to start looking (1-indexed)
+
+    Returns:
+        Docstring content or None if not found
+    """
+    if start_line > len(lines):
+        return None
+
+    start_idx = start_line - 1
+
+    # Check for Python-style docstring
+    triple_double = '"""'
+    triple_single = "'''"
+    for quote in [triple_double, triple_single]:
+        if quote in lines[start_idx]:
+            # Multi-line docstring
+            docstring_lines = []
+            in_docstring = False
+
+            for line in lines[start_idx:]:
+                if quote in line:
+                    if in_docstring:
+                        # End of docstring
+                        docstring_lines.append(line[: line.index(quote) + 3])
+                        break
+                    else:
+                        # Start of docstring
+                        in_docstring = True
+                        docstring_lines.append(line)
+                        if line.count(quote) >= 2:
+                            # Single-line docstring
+                            break
+                elif in_docstring:
+                    docstring_lines.append(line)
+
+            if docstring_lines:
+                return "".join(docstring_lines).strip()
+
+    # Check for JavaDoc-style comment
+    if start_idx > 0 and "/**" in lines[start_idx - 1]:
+        comment_lines = []
+        for i in range(start_idx - 1, -1, -1):
+            comment_lines.insert(0, lines[i])
+            if "/**" in lines[i]:
+                break
+
+        for i in range(start_idx, len(lines)):
+            if "*/" in lines[i]:
+                comment_lines.append(lines[i])
+                break
+            comment_lines.append(lines[i])
+
+        return "".join(comment_lines).strip()
+
+    # Check for hash/slash comments on previous lines
+    comment_lines = []
+    for i in range(start_idx - 1, -1, -1):
+        line = lines[i].strip()
+        if line.startswith("#") or line.startswith("//"):
+            comment_lines.insert(0, lines[i])
+        elif line:
+            break
+
+    if comment_lines:
+        return "".join(comment_lines).strip()
+
+    return None
+
+
+def extract_imports_with_pattern(
+    content: str, pattern: Pattern[str], chunk_type: str = "import"
+) -> list[str]:
+    """Extract import/require/use statements using a regex pattern.
+
+    Args:
+        content: Source code content
+        pattern: Compiled regex pattern to match imports
+        chunk_type: Type of import (import, require, use, etc.)
+
+    Returns:
+        List of import statements
+    """
+    imports = []
+    for match in pattern.finditer(content):
+        import_line = match.group(0).strip()
+        imports.append(import_line)
+    return imports
+
+
+def find_code_blocks_with_patterns(
+    content: str, lines: list[str], patterns: dict[str, Pattern[str]], file_path: Path
+) -> list[CodeChunk]:
+    """Find code blocks (functions, classes, etc.) using regex patterns.
+
+    This is a generic fallback parser that can be configured with different
+    patterns for different languages.
+
+    Args:
+        content: Source code content
+        lines: Pre-split lines
+        patterns: Dictionary mapping block types to compiled regex patterns
+        file_path: Path to source file
+
+    Returns:
+        List of code chunks
+    """
+    chunks = []
+
+    for block_type, pattern in patterns.items():
+        for match in pattern.finditer(content):
+            # Extract the name from the first capturing group
+            name = match.group(1) if match.groups() else "unknown"
+
+            # Find line number
+            match_pos = match.start()
+            start_line = content[:match_pos].count("\n") + 1
+
+            # Find end of block using indentation
+            end_line = find_block_end(lines, start_line)
+
+            # Get block content
+            block_content = get_line_range(lines, start_line, end_line)
+
+            if block_content.strip():
+                # Extract docstring
+                docstring = extract_docstring(lines, start_line + 1)
+
+                chunk = CodeChunk(
+                    content=block_content,
+                    start_line=start_line,
+                    end_line=end_line,
+                    file_path=str(file_path),
+                    chunk_type=block_type,
+                    metadata={
+                        "name": name,
+                        "docstring": docstring,
+                        "source": "regex_fallback",
+                    },
+                )
+                chunks.append(chunk)
+
+    return chunks

mcp_vector_search/py.typed

@@ -0,0 +1 @@
+# PEP 561 marker file for type information

mcp_vector_search/utils/__init__.py

@@ -0,0 +1,42 @@
+"""Utility modules for MCP Vector Search."""
+
+from .gitignore import (
+    GitignoreParser,
+    GitignorePattern,
+    create_gitignore_parser,
+    is_path_gitignored,
+)
+from .gitignore_updater import ensure_gitignore_entry
+from .timing import (
+    PerformanceProfiler,
+    SearchProfiler,
+    TimingResult,
+    get_global_profiler,
+    print_global_report,
+    time_async_block,
+    time_block,
+    time_function,
+)
+from .version import get_user_agent, get_version_info, get_version_string
+
+__all__ = [
+    # Gitignore utilities
+    "GitignoreParser",
+    "GitignorePattern",
+    "create_gitignore_parser",
+    "is_path_gitignored",
+    "ensure_gitignore_entry",
+    # Timing utilities
+    "PerformanceProfiler",
+    "TimingResult",
+    "time_function",
+    "time_block",
+    "time_async_block",
+    "get_global_profiler",
+    "print_global_report",
+    "SearchProfiler",
+    # Version utilities
+    "get_version_info",
+    "get_version_string",
+    "get_user_agent",
+]