mcp-vector-search 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

mcp_vector_search/__init__.py

@@ -1,7 +1,7 @@
 """MCP Vector Search - CLI-first semantic code search with MCP integration."""
 
-__version__ = "0.5.0"
-__build__ = "18"
+__version__ = "0.5.1"
+__build__ = "19"
 __author__ = "Robert Matsuoka"
 __email__ = "bobmatnyc@gmail.com"
 

mcp_vector_search/parsers/__init__.py

@@ -1,7 +1,8 @@
 """Language parsers for MCP Vector Search."""
 
 from .dart import DartParser
+from .html import HTMLParser
 from .php import PHPParser
 from .ruby import RubyParser
 
-__all__ = ["DartParser", "PHPParser", "RubyParser"]
+__all__ = ["DartParser", "HTMLParser", "PHPParser", "RubyParser"]

mcp_vector_search/parsers/html.py

@@ -0,0 +1,413 @@
+"""HTML parser for MCP Vector Search."""
+
+from html.parser import HTMLParser as BaseHTMLParser
+from pathlib import Path
+
+from ..core.models import CodeChunk
+from .base import BaseParser
+
+
+class HTMLContentParser(BaseHTMLParser):
+    """HTML parser for extracting semantic content from HTML documents.
+
+    Extracts meaningful content from semantic HTML tags while ignoring
+    scripts, styles, and other non-content elements.
+    """
+
+    def __init__(self) -> None:
+        """Initialize the HTML content parser."""
+        super().__init__()
+        self.sections: list[dict] = []
+        self.current_section: dict | None = None
+        self.current_tag: str | None = None
+        self.current_attrs: dict = {}
+        self.tag_stack: list[tuple[str, dict]] = []
+        self.ignore_content = False
+        self.line_number = 1
+
+        # Semantic tags that define sections
+        self.section_tags = {
+            "h1",
+            "h2",
+            "h3",
+            "h4",
+            "h5",
+            "h6",
+            "section",
+            "article",
+            "main",
+            "aside",
+            "header",
+            "footer",
+            "nav",
+        }
+        # Tags to ignore completely
+        self.ignore_tags = {"script", "style", "noscript"}
+        # Inline text tags
+        self.inline_tags = {
+            "span",
+            "a",
+            "strong",
+            "em",
+            "b",
+            "i",
+            "code",
+            "pre",
+            "small",
+        }
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        """Handle opening HTML tags.
+
+        Args:
+            tag: Tag name
+            attrs: List of (name, value) tuples for tag attributes
+        """
+        tag = tag.lower()
+        attr_dict = {k: v for k, v in attrs if v is not None}
+
+        # Track tag stack
+        self.tag_stack.append((tag, attr_dict))
+
+        # Ignore script/style content
+        if tag in self.ignore_tags:
+            self.ignore_content = True
+            return
+
+        # Start new section for semantic tags
+        if tag in self.section_tags:
+            # Save current section if exists
+            if self.current_section and self.current_section.get("content", "").strip():
+                self._finalize_current_section()
+
+            # Start new section
+            tag_id = attr_dict.get("id", "")
+            tag_class = attr_dict.get("class", "")
+
+            # Create tag identifier
+            tag_info = tag
+            if tag_id:
+                tag_info = f"{tag}#{tag_id}"
+            elif tag_class:
+                tag_info = f"{tag}.{tag_class.split()[0]}"
+
+            self.current_section = {
+                "tag": tag,
+                "tag_info": tag_info,
+                "content": "",
+                "start_line": self.getpos()[0],
+                "attrs": attr_dict,
+            }
+
+        # Handle paragraph tags
+        if tag == "p":
+            if not self.current_section:
+                # Create implicit section for orphan paragraph
+                self.current_section = {
+                    "tag": "p",
+                    "tag_info": "p",
+                    "content": "",
+                    "start_line": self.getpos()[0],
+                    "attrs": {},
+                }
+
+        self.current_tag = tag
+
+    def handle_endtag(self, tag: str) -> None:
+        """Handle closing HTML tags.
+
+        Args:
+            tag: Tag name
+        """
+        tag = tag.lower()
+
+        # Pop from stack
+        if self.tag_stack and self.tag_stack[-1][0] == tag:
+            self.tag_stack.pop()
+
+        # Stop ignoring content after script/style
+        if tag in self.ignore_tags:
+            self.ignore_content = False
+            return
+
+        # Finalize section for semantic tags
+        if tag in self.section_tags or tag == "p":
+            if self.current_section and self.current_section.get("content", "").strip():
+                self.current_section["end_line"] = self.getpos()[0]
+                self._finalize_current_section()
+
+    def handle_data(self, data: str) -> None:
+        """Handle text content between tags.
+
+        Args:
+            data: Text content
+        """
+        if self.ignore_content or not data.strip():
+            return
+
+        # Add content to current section
+        if self.current_section is not None:
+            # Add spacing between inline elements
+            if self.current_section["content"] and not self.current_section[
+                "content"
+            ].endswith(" "):
+                self.current_section["content"] += " "
+            self.current_section["content"] += data.strip()
+
+    def _finalize_current_section(self) -> None:
+        """Finalize and save the current section."""
+        if not self.current_section:
+            return
+
+        content = self.current_section["content"].strip()
+
+        # Only save sections with meaningful content (min 20 chars)
+        if len(content) >= 20:
+            self.sections.append(
+                {
+                    "tag": self.current_section["tag"],
+                    "tag_info": self.current_section["tag_info"],
+                    "content": content,
+                    "start_line": self.current_section["start_line"],
+                    "end_line": self.current_section.get(
+                        "end_line", self.current_section["start_line"]
+                    ),
+                    "attrs": self.current_section["attrs"],
+                }
+            )
+
+        self.current_section = None
+
+    def get_sections(self) -> list[dict]:
+        """Get all extracted sections.
+
+        Returns:
+            List of section dictionaries
+        """
+        # Finalize any remaining section
+        if self.current_section and self.current_section.get("content", "").strip():
+            self._finalize_current_section()
+
+        return self.sections
+
+
+class HTMLParser(BaseParser):
+    """Parser for HTML files (.html, .htm).
+
+    Extracts semantic content from HTML documents by parsing
+    heading hierarchy, sections, articles, and paragraphs.
+    """
+
+    def __init__(self) -> None:
+        """Initialize HTML parser."""
+        super().__init__("html")
+
+    async def parse_file(self, file_path: Path) -> list[CodeChunk]:
+        """Parse an HTML file and extract semantic chunks.
+
+        Args:
+            file_path: Path to the HTML file
+
+        Returns:
+            List of semantic content chunks
+        """
+        try:
+            with open(file_path, encoding="utf-8", errors="ignore") as f:
+                content = f.read()
+            return await self.parse_content(content, file_path)
+        except Exception:
+            # Return empty list if file can't be read
+            return []
+
+    async def parse_content(self, content: str, file_path: Path) -> list[CodeChunk]:
+        """Parse HTML content into semantic chunks.
+
+        Extracts content from semantic HTML tags (headings, sections, articles)
+        while ignoring scripts, styles, and other non-content elements.
+
+        Args:
+            content: HTML content to parse
+            file_path: Path to the source file
+
+        Returns:
+            List of semantic content chunks
+        """
+        if not content.strip():
+            return []
+
+        # Parse HTML content
+        parser = HTMLContentParser()
+        try:
+            parser.feed(content)
+        except Exception:
+            # If parsing fails, fall back to simple text extraction
+            return await self._fallback_parse(content, file_path)
+
+        sections = parser.get_sections()
+
+        if not sections:
+            # No semantic sections found, try fallback
+            return await self._fallback_parse(content, file_path)
+
+        # Convert sections to chunks
+        chunks = []
+
+        # Merge small sections for better semantic coherence
+        merged_sections = self._merge_small_sections(sections)
+
+        for section in merged_sections:
+            chunk_type = self._get_chunk_type(section["tag"])
+
+            # Create descriptive metadata
+            metadata = {
+                "chunk_type": chunk_type,
+                "function_name": section["tag_info"],  # Use tag_info as identifier
+            }
+
+            # Add class name for sections with specific IDs
+            if section["attrs"].get("id"):
+                metadata["class_name"] = section["attrs"]["id"]
+
+            chunk = self._create_chunk(
+                content=section["content"],
+                file_path=file_path,
+                start_line=section["start_line"],
+                end_line=section["end_line"],
+                **metadata,
+            )
+            chunks.append(chunk)
+
+        return chunks
+
+    def _get_chunk_type(self, tag: str) -> str:
+        """Determine chunk type based on HTML tag.
+
+        Args:
+            tag: HTML tag name
+
+        Returns:
+            Chunk type string
+        """
+        if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}:
+            return "heading"
+        elif tag in {"section", "article", "main"}:
+            return "section"
+        elif tag == "p":
+            return "paragraph"
+        else:
+            return "content"
+
+    def _merge_small_sections(
+        self, sections: list[dict], target_size: int = 200, max_size: int = 500
+    ) -> list[dict]:
+        """Merge small sections to create optimal chunk sizes.
+
+        Args:
+            sections: List of section dictionaries
+            target_size: Target minimum size for chunks in characters
+            max_size: Maximum size for chunks in characters
+
+        Returns:
+            List of merged section dictionaries
+        """
+        if not sections:
+            return []
+
+        merged = []
+        current_merge: dict | None = None
+
+        for section in sections:
+            section_len = len(section["content"])
+
+            # Always start new section for h1 tags
+            if section["tag"] == "h1":
+                if current_merge:
+                    merged.append(current_merge)
+                current_merge = section.copy()
+                continue
+
+            if current_merge is None:
+                current_merge = section.copy()
+            elif len(current_merge["content"]) + section_len < max_size:
+                # Merge with current if under max size
+                current_merge["content"] += "\n\n" + section["content"]
+                current_merge["end_line"] = section["end_line"]
+
+                # Update tag_info to reflect merged content
+                if current_merge["tag_info"] != section["tag_info"]:
+                    current_merge["tag_info"] = (
+                        f"{current_merge['tag_info']}+{section['tag_info']}"
+                    )
+            else:
+                # Start new section if max size would be exceeded
+                if len(current_merge["content"]) >= target_size:
+                    merged.append(current_merge)
+                current_merge = section.copy()
+
+        # Add last section
+        if current_merge and len(current_merge["content"]) >= 20:
+            merged.append(current_merge)
+
+        return merged
+
+    async def _fallback_parse(self, content: str, file_path: Path) -> list[CodeChunk]:
+        """Fallback parsing for malformed HTML.
+
+        Strips HTML tags and creates simple text chunks.
+
+        Args:
+            content: HTML content
+            file_path: Path to source file
+
+        Returns:
+            List of text chunks
+        """
+        # Simple HTML tag removal
+        import re
+
+        # Remove script and style tags with content
+        content = re.sub(
+            r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL | re.IGNORECASE
+        )
+        content = re.sub(
+            r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL | re.IGNORECASE
+        )
+
+        # Remove HTML tags
+        content = re.sub(r"<[^>]+>", " ", content)
+
+        # Normalize whitespace
+        content = re.sub(r"\s+", " ", content)
+
+        # Split into chunks
+        lines = content.split(".")
+        chunks = []
+        current_chunk = []
+        start_line = 1
+
+        for i, line in enumerate(lines, 1):
+            current_chunk.append(line)
+            chunk_text = ". ".join(current_chunk)
+
+            if len(chunk_text) >= 200 or i == len(lines):
+                if chunk_text.strip():
+                    chunk = self._create_chunk(
+                        content=chunk_text.strip(),
+                        file_path=file_path,
+                        start_line=start_line,
+                        end_line=i,
+                        chunk_type="text",
+                    )
+                    chunks.append(chunk)
+                    current_chunk = []
+                    start_line = i + 1
+
+        return chunks
+
+    def get_supported_extensions(self) -> list[str]:
+        """Get list of supported file extensions.
+
+        Returns:
+            List of supported extensions
+        """
+        return [".html", ".htm"]

mcp_vector_search/parsers/registry.py

@@ -6,6 +6,7 @@ from loguru import logger
 
 from .base import BaseParser, FallbackParser
 from .dart import DartParser
+from .html import HTMLParser
 from .javascript import JavaScriptParser, TypeScriptParser
 from .php import PHPParser
 from .python import PythonParser
@@ -59,6 +60,10 @@ class ParserRegistry:
         text_parser = TextParser()
         self.register_parser("text", text_parser)
 
+        # Register HTML parser for .html files
+        html_parser = HTMLParser()
+        self.register_parser("html", html_parser)
+
     def register_parser(self, language: str, parser: BaseParser) -> None:
         """Register a parser for a specific language.
 

mcp_vector_search/parsers/text.py

@@ -7,7 +7,7 @@ from .base import BaseParser
 
 
 class TextParser(BaseParser):
-    """Parser for plain text files (.txt)."""
+    """Parser for plain text and markdown files (.txt, .md, .markdown)."""
 
     def __init__(self) -> None:
         """Initialize text parser."""
@@ -182,4 +182,4 @@ class TextParser(BaseParser):
         Returns:
             List of supported extensions
         """
-        return [".txt"]
+        return [".txt", ".md", ".markdown"]

{mcp_vector_search-0.5.0.dist-info → mcp_vector_search-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcp-vector-search
-Version: 0.5.0
+Version: 0.5.1
 Summary: CLI-first semantic code search with MCP integration
 Project-URL: Homepage, https://github.com/bobmatnyc/mcp-vector-search
 Project-URL: Documentation, https://mcp-vector-search.readthedocs.io
@@ -71,7 +71,7 @@ A modern, fast, and intelligent code search tool that understands your codebase
 ### 🚀 **Core Capabilities**
 - **Semantic Search**: Find code by meaning, not just keywords
 - **AST-Aware Parsing**: Understands code structure (functions, classes, methods)
-- **Multi-Language Support**: Python, JavaScript, TypeScript, Dart/Flutter, PHP, Ruby (with extensible architecture)
+- **Multi-Language Support**: 8 languages - Python, JavaScript, TypeScript, Dart/Flutter, PHP, Ruby, HTML, and Markdown/Text (with extensible architecture)
 - **Real-time Indexing**: File watching with automatic index updates
 - **Local-First**: Complete privacy with on-device processing
 - **Zero Configuration**: Auto-detects project structure and languages
@@ -355,21 +355,35 @@ Projects are configured via `.mcp-vector-search/config.json`:
 
 ### Supported Languages
 
+MCP Vector Search supports **8 programming languages** with full semantic search capabilities:
+
+| Language   | Extensions | Status | Features |
+|------------|------------|--------|----------|
+| Python     | `.py`, `.pyw` | ✅ Full | Functions, classes, methods, docstrings |
+| JavaScript | `.js`, `.jsx`, `.mjs` | ✅ Full | Functions, classes, JSDoc, ES6+ syntax |
+| TypeScript | `.ts`, `.tsx` | ✅ Full | Interfaces, types, generics, decorators |
+| Dart       | `.dart` | ✅ Full | Functions, classes, widgets, async, dartdoc |
+| PHP        | `.php`, `.phtml` | ✅ Full | Classes, methods, traits, PHPDoc, Laravel patterns |
+| Ruby       | `.rb`, `.rake`, `.gemspec` | ✅ Full | Modules, classes, methods, RDoc, Rails patterns |
+| HTML       | `.html`, `.htm` | ✅ Full | Semantic content extraction, heading hierarchy, text chunking |
+| Text/Markdown | `.txt`, `.md`, `.markdown` | ✅ Basic | Semantic chunking for documentation |
+
+**Planned Languages:**
 | Language   | Status | Features |
 |------------|--------|----------|
-| Python     | ✅ Full | Functions, classes, methods, docstrings |
-| JavaScript | ✅ Full | Functions, classes, JSDoc, ES6+ syntax |
-| TypeScript | ✅ Full | Interfaces, types, generics, decorators |
-| Dart       | ✅ Full | Functions, classes, widgets, async, dartdoc |
-| PHP        | ✅ Full | Classes, methods, traits, PHPDoc, Laravel patterns |
-| Ruby       | ✅ Full | Modules, classes, methods, RDoc, Rails patterns |
-| Text/Markdown | ✅ Basic | Semantic chunking for documentation |
 | Java       | 🔄 Planned | Classes, methods, annotations |
 | Go         | 🔄 Planned | Functions, structs, interfaces |
 | Rust       | 🔄 Planned | Functions, structs, traits |
 
 #### New Language Support
 
+**HTML Support** (Unreleased):
+- **Semantic Extraction**: Content from h1-h6, p, section, article, main, aside, nav, header, footer
+- **Intelligent Chunking**: Based on heading hierarchy (h1-h6)
+- **Context Preservation**: Maintains class and id attributes for searchability
+- **Script/Style Filtering**: Ignores non-content elements
+- **Use Cases**: Static sites, documentation, web templates, HTML fragments
+
 **Dart/Flutter Support** (v0.4.15):
 - **Widget Detection**: StatelessWidget, StatefulWidget recognition
 - **State Classes**: Automatic parsing of `_WidgetNameState` patterns

{mcp_vector_search-0.5.0.dist-info → mcp_vector_search-0.5.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-mcp_vector_search/__init__.py,sha256=eis7op2vTtnKT4n7nV7zQEHKVUChwmbCwamTLVHDzIU,299
+mcp_vector_search/__init__.py,sha256=wWzwwtd5LyUT8xL6wgwEixYc6hnUgUWgjsUwYPE6ub8,299
 mcp_vector_search/py.typed,sha256=lCKeV9Qcn9sGtbRsgg-LJO2ZwWRuknnnlmomq3bJFH0,43
 mcp_vector_search/cli/__init__.py,sha256=TNB7CaOASz8u3yHWLbNmo8-GtHF0qwUjVKWAuNphKgo,40
 mcp_vector_search/cli/didyoumean.py,sha256=dzHtzGaCRtKGum1KqyPBNP4_7d2JNsxgcT4kD7rIfp4,15941
@@ -39,21 +39,22 @@ mcp_vector_search/core/watcher.py,sha256=-DFRCnuUfcqcTrkZPQqfJSvxKAxnpt-axgEj1V-
 mcp_vector_search/mcp/__init__.py,sha256=gfKR0QV7Jqvj5y0LMBe9gSghd5_rPsvm_rml0ryQtoY,158
 mcp_vector_search/mcp/__main__.py,sha256=KgwB59HM5pRLe2Aj-fvDFcTp95lyT0wfmS3ENcx9gPc,571
 mcp_vector_search/mcp/server.py,sha256=YmHyvJqg_CjxEN356ShFrTPLgDKzaLXyrt8tNHVryEY,28322
-mcp_vector_search/parsers/__init__.py,sha256=Qza-RapcVLhBpcuL2CCmf0b5LSzig1ZfhiCOZ5esO8w,185
+mcp_vector_search/parsers/__init__.py,sha256=jr0Yqz1xMok4lnG7_aXnkZThGuefrlAj8PWVbfeT3QQ,228
 mcp_vector_search/parsers/base.py,sha256=-zBY9T0modfegowaNyf5_upkS3ImR4TgrRwoSLuAiDw,5421
 mcp_vector_search/parsers/dart.py,sha256=Cs10Tw3XvvA_nJpUilKgrj8E2Sq6F3urODEFH_ZPS3M,21840
+mcp_vector_search/parsers/html.py,sha256=nzEVDV4oCBp3wpL8vp6WWx5eqiB39agu9E048JkuRJQ,13010
 mcp_vector_search/parsers/javascript.py,sha256=P7fT_tXCzUuXATTkTx_DyD4EuG0_KjIDlay09MhkKTE,9824
 mcp_vector_search/parsers/php.py,sha256=1QjnE8SAQF86VQ7pNfn1Pmpg5Dni4M7KCLU7212DkXM,24774
 mcp_vector_search/parsers/python.py,sha256=IB4gQ6DD6Oqvws5p3LQtHTJOtmH6f9FWmO9fQRfGQx4,15688
-mcp_vector_search/parsers/registry.py,sha256=HN2v4p3UsUzplnKnOwuV89KfE70VZBc9ymB0Vypq9bU,6295
+mcp_vector_search/parsers/registry.py,sha256=L00EUp58ff_xatgQW-cBvE0AVBQi7cGtvr6zWV_NYbc,6457
 mcp_vector_search/parsers/ruby.py,sha256=xNn_z8txAWL7E1ULcFMiqn5idFhf5GQn8N3x1yE-c2k,23818
-mcp_vector_search/parsers/text.py,sha256=dUxRzi-aoNsWP_fHRjkeCBZU0CMhnvBOwEuOHnPJKyc,5975
+mcp_vector_search/parsers/text.py,sha256=qKD-P76-lVjY16FH9ZffKihWaphoBV2R8laWXOwoNug,6024
 mcp_vector_search/utils/__init__.py,sha256=Eq6lY-oPMfCt-GpPUbg9QbmTHuQVmTaVDBMU2183KVw,887
 mcp_vector_search/utils/gitignore.py,sha256=lc8_GK2L8dzlqPRZNB3XfQvVCbnLtz_l0eQN_13z8Xo,7684
 mcp_vector_search/utils/timing.py,sha256=THC7mfbTYnUpnnDcblgQacYMzbEkfFoIShx6plmhCgg,11285
 mcp_vector_search/utils/version.py,sha256=d7fS-CLemxb8UzZ9j18zH0Y0Ud097ljKKYYOPulnGPE,1138
-mcp_vector_search-0.5.0.dist-info/METADATA,sha256=p_JJwOELJxBDp0AkQcizMKkk3nVNmUQKuZ1nKpD2SSw,18104
-mcp_vector_search-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-mcp_vector_search-0.5.0.dist-info/entry_points.txt,sha256=y3Ygtc_JiBchNEIL-tPABo7EbzBExGAxwGdkkeP5D2I,86
-mcp_vector_search-0.5.0.dist-info/licenses/LICENSE,sha256=FqZUgGJH_tZKZLQsMCpXaLawRyLmyFKRVfMwYyEcyTs,1072
-mcp_vector_search-0.5.0.dist-info/RECORD,,
+mcp_vector_search-0.5.1.dist-info/METADATA,sha256=6QMMzeEpShXc8-jF74xNpBEuNGbexHDpNZ5AqxsPY88,19021
+mcp_vector_search-0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+mcp_vector_search-0.5.1.dist-info/entry_points.txt,sha256=y3Ygtc_JiBchNEIL-tPABo7EbzBExGAxwGdkkeP5D2I,86
+mcp_vector_search-0.5.1.dist-info/licenses/LICENSE,sha256=FqZUgGJH_tZKZLQsMCpXaLawRyLmyFKRVfMwYyEcyTs,1072
+mcp_vector_search-0.5.1.dist-info/RECORD,,