PyPI - iflow-mcp_anton-prosterity-documentation-search-enhanced - Versions diffs - 1.9.0__py3-none-any.whl - Mend

iflow-mcp_anton-prosterity-documentation-search-enhanced 1.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (26) hide show

documentation_search_enhanced/__init__.py +14 -0
documentation_search_enhanced/__main__.py +6 -0
documentation_search_enhanced/config.json +1674 -0
documentation_search_enhanced/config_manager.py +233 -0
documentation_search_enhanced/config_validator.py +79 -0
documentation_search_enhanced/content_enhancer.py +578 -0
documentation_search_enhanced/docker_manager.py +87 -0
documentation_search_enhanced/logger.py +179 -0
documentation_search_enhanced/main.py +2170 -0
documentation_search_enhanced/project_generator.py +260 -0
documentation_search_enhanced/project_scanner.py +85 -0
documentation_search_enhanced/reranker.py +230 -0
documentation_search_enhanced/site_index_builder.py +274 -0
documentation_search_enhanced/site_index_downloader.py +222 -0
documentation_search_enhanced/site_search.py +1325 -0
documentation_search_enhanced/smart_search.py +473 -0
documentation_search_enhanced/snyk_integration.py +657 -0
documentation_search_enhanced/vector_search.py +303 -0
documentation_search_enhanced/version_resolver.py +189 -0
documentation_search_enhanced/vulnerability_scanner.py +545 -0
documentation_search_enhanced/web_scraper.py +117 -0
iflow_mcp_anton_prosterity_documentation_search_enhanced-1.9.0.dist-info/METADATA +195 -0
iflow_mcp_anton_prosterity_documentation_search_enhanced-1.9.0.dist-info/RECORD +26 -0
iflow_mcp_anton_prosterity_documentation_search_enhanced-1.9.0.dist-info/WHEEL +4 -0
iflow_mcp_anton_prosterity_documentation_search_enhanced-1.9.0.dist-info/entry_points.txt +2 -0
iflow_mcp_anton_prosterity_documentation_search_enhanced-1.9.0.dist-info/licenses/LICENSE +21 -0

documentation_search_enhanced/project_generator.py ADDED Viewed

@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+"""
+Generates boilerplate code and file structures for new projects.
+"""
+import os
+from typing import Dict, List, TypedDict, NotRequired
+# --- Project Templates ---
+TEMPLATES: Dict[str, Dict[str, str]] = {
+    "fastapi": {
+        "main.py": """
+from fastapi import FastAPI
+app = FastAPI(
+    title="My FastAPI Project",
+    description="A new project generated by MCP.",
+    version="0.1.0",
+)
+@app.get("/")
+async def read_root():
+    return {"message": "Hello, World!"}
+@app.get("/items/{item_id}")
+async def read_item(item_id: int, q: str | None = None):
+    return {"item_id": item_id, "q": q}
+""",
+        "pyproject.toml": """
+[project]
+name = "PROJECT_NAME_PLACEHOLDER"
+version = "0.1.0"
+description = "A new FastAPI project."
+authors = [{ name = "Your Name", email = "you@example.com" }]
+requires-python = ">=3.12"
+dependencies = [
+    "fastapi",
+    "uvicorn[standard]",
+]
+[project.optional-dependencies]
+dev = ["pytest"]
+""",
+        "README.md": """
+# PROJECT_NAME_PLACEHOLDER
+A new FastAPI project generated by Documentation Search Enhanced MCP.
+## To run:
+1. `uv pip sync`
+2. `uv run uvicorn main:app --reload`
+## To test:
+`uv run pytest`
+""",
+        ".gitignore": """
+__pycache__/
+*.pyc
+.env
+.venv/
+dist/
+build/
+*.egg-info
+""",
+        "tests/test_main.py": """
+from fastapi.testclient import TestClient
+from main import app
+client = TestClient(app)
+def test_read_root():
+    response = client.get("/")
+    assert response.status_code == 200
+    assert response.json() == {"message": "Hello, World!"}
+""",
+    },
+    "react-vite": {
+        "index.html": """
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>PROJECT_NAME_PLACEHOLDER</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>
+""",
+        "package.json": """
+{{
+  "name": "PROJECT_NAME_PLACEHOLDER",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {{
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  }},
+  "dependencies": {{
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  }},
+  "devDependencies": {{
+    "@types/react": "^18.2.15",
+    "@types/react-dom": "^18.2.7",
+    "@vitejs/plugin-react": "^4.0.3",
+    "eslint": "^8.45.0",
+    "eslint-plugin-react": "^7.32.2",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.3",
+    "vite": "^4.4.5"
+  }}
+}}
+""",
+        "vite.config.js": """
+import { defineConfig } from 'vite'
+import react from '@vitejs/plugin-react'
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [react()],
+})
+""",
+        ".gitignore": """
+# Logs
+logs
+*.log
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+# Dependency directories
+node_modules/
+dist/
+# IDE files
+.idea/
+.vscode/
+# Environment variables
+.env
+.env.local
+""",
+        "src/App.jsx": """
+import './App.css'
+function App() {
+  return (
+    <>
+      <h1>PROJECT_NAME_PLACEHOLDER</h1>
+      <p className="read-the-docs">
+        React + Vite project generated by MCP.
+      </p>
+    </>
+  )
+}
+export default App
+""",
+        "src/main.jsx": """
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import App from './App.jsx'
+import './index.css'
+ReactDOM.createRoot(document.getElementById('root')).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+)
+""",
+        "src/index.css": """
+:root {
+  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
+}
+""",
+        "src/App.css": """
+#root {
+  max-width: 1280px;
+  margin: 0 auto;
+  padding: 2rem;
+  text-align: center;
+}
+""",
+    },
+}
+class ProjectCreationSummary(TypedDict):
+    project_name: str
+    template_used: str
+    project_path: str
+    directories_created: List[str]
+    files_created: List[str]
+    user_summary: NotRequired[str]
+def generate_project(
+    project_name: str, template_name: str, base_path: str = "."
+) -> ProjectCreationSummary:
+    """
+    Generates a new project from a template.
+    Args:
+        project_name: The name of the new project (will be created as a directory).
+        template_name: The name of the template to use (e.g., 'fastapi').
+        base_path: The path where the project directory will be created.
+    Returns:
+        A dictionary summarizing the created files and directories.
+    """
+    if template_name not in TEMPLATES:
+        raise ValueError(
+            f"Template '{template_name}' not found. Available templates: {list(TEMPLATES.keys())}"
+        )
+    project_path = os.path.join(base_path, project_name)
+    if os.path.exists(project_path):
+        raise FileExistsError(f"Directory '{project_path}' already exists.")
+    os.makedirs(project_path)
+    template = TEMPLATES[template_name]
+    created_files = []
+    created_dirs = {project_path}
+    for file_path, content in template.items():
+        # Handle nested directories
+        full_path = os.path.join(project_path, file_path)
+        dir_name = os.path.dirname(full_path)
+        if not os.path.exists(dir_name):
+            os.makedirs(dir_name)
+            created_dirs.add(dir_name)
+        # Replace project name placeholder
+        formatted_content = content.replace("PROJECT_NAME_PLACEHOLDER", project_name)
+        with open(full_path, "w", encoding="utf-8") as f:
+            f.write(formatted_content.strip())
+        created_files.append(full_path)
+    return {
+        "project_name": project_name,
+        "template_used": template_name,
+        "project_path": project_path,
+        "directories_created": sorted(list(created_dirs)),
+        "files_created": sorted(created_files),
+    }

documentation_search_enhanced/project_scanner.py ADDED Viewed

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""
+Scans project directories to find and parse dependency files.
+"""
+import os
+import json
+import sys
+from typing import Dict, Tuple, Optional
+import re
+import tomllib
+def _parse_requirement(req: str) -> Tuple[str, str]:
+    """Parses a requirement string (e.g., 'fastapi==0.1.0' or 'django>=3.2')."""
+    match = re.match(r"([a-zA-Z0-9\-_]+)\s*([~<>=!]=?)\s*([0-9\.\*a-zA-Z]+)", req)
+    if match:
+        name, specifier, version = match.groups()
+        return name.strip(), f"{specifier}{version}"
+    return req.strip(), "latest"
+def parse_pyproject_toml(content: str) -> Dict[str, str]:
+    """Parses dependencies from pyproject.toml content."""
+    data = tomllib.loads(content)
+    dependencies = data.get("project", {}).get("dependencies", [])
+    parsed_deps = {}
+    for req in dependencies:
+        name, version = _parse_requirement(req)
+        parsed_deps[name] = version
+    return parsed_deps
+def parse_requirements_txt(content: str) -> Dict[str, str]:
+    """Parses dependencies from requirements.txt content."""
+    lines = content.splitlines()
+    parsed_deps = {}
+    for line in lines:
+        line = line.strip()
+        if line and not line.startswith("#"):
+            name, version = _parse_requirement(line)
+            parsed_deps[name] = version
+    return parsed_deps
+def parse_package_json(content: str) -> Dict[str, str]:
+    """Parses dependencies from package.json content."""
+    data = json.loads(content)
+    deps = data.get("dependencies", {})
+    dev_deps = data.get("devDependencies", {})
+    deps.update(dev_deps)
+    return deps
+def find_and_parse_dependencies(
+    directory: str,
+) -> Optional[Tuple[str, str, Dict[str, str]]]:
+    """
+    Finds and parses the most relevant dependency file in a directory.
+    Returns:
+        A tuple of (file_path, ecosystem, dependencies_dict) or None.
+    """
+    supported_files = {
+        "pyproject.toml": ("PyPI", parse_pyproject_toml),
+        "requirements.txt": ("PyPI", parse_requirements_txt),
+        "package.json": ("npm", parse_package_json),
+    }
+    for filename, (ecosystem, parser_func) in supported_files.items():
+        file_path = os.path.join(directory, filename)
+        if os.path.exists(file_path):
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    content = f.read()
+                dependencies = parser_func(content)
+                return filename, ecosystem, dependencies
+            except Exception as e:
+                print(f"⚠️ Error parsing {filename}: {e}", file=sys.stderr)
+                # Continue to the next file type if parsing fails
+                continue
+    return None

documentation_search_enhanced/reranker.py ADDED Viewed

@@ -0,0 +1,230 @@
+"""Search result reranking using hybrid scoring (vector + keyword + metadata)."""
+import logging
+import re
+from typing import List, Optional
+from .vector_search import get_vector_engine
+from .smart_search import SearchResult
+logger = logging.getLogger(__name__)
+class SearchReranker:
+    """
+    Rerank search results using a hybrid scoring approach:
+    - Semantic similarity (vector embeddings): 50% weight
+    - Keyword matching relevance: 30% weight
+    - Source authority/freshness: 20% weight
+    """
+    def __init__(
+        self,
+        semantic_weight: float = 0.5,
+        keyword_weight: float = 0.3,
+        metadata_weight: float = 0.2,
+    ):
+        """
+        Initialize the reranker with configurable weights.
+        Args:
+            semantic_weight: Weight for vector similarity score (0-1)
+            keyword_weight: Weight for keyword matching score (0-1)
+            metadata_weight: Weight for metadata scoring (0-1)
+        """
+        self.semantic_weight = semantic_weight
+        self.keyword_weight = keyword_weight
+        self.metadata_weight = metadata_weight
+        # Ensure weights sum to 1.0
+        total = semantic_weight + keyword_weight + metadata_weight
+        if abs(total - 1.0) > 0.01:
+            logger.warning(f"Reranker weights sum to {total}, normalizing to 1.0")
+            self.semantic_weight /= total
+            self.keyword_weight /= total
+            self.metadata_weight /= total
+        self.vector_engine = get_vector_engine()
+    async def rerank(
+        self,
+        results: List[SearchResult],
+        query: str,
+        use_semantic: bool = True,
+    ) -> List[SearchResult]:
+        """
+        Rerank search results using hybrid scoring.
+        Args:
+            results: List of search results to rerank
+            query: Original search query
+            use_semantic: Whether to use semantic scoring (can be disabled for speed)
+        Returns:
+            Reranked list of search results
+        """
+        if not results:
+            return results
+        logger.debug(f"Reranking {len(results)} results for query: {query[:50]}...")
+        # Calculate scores for each result
+        scored_results = []
+        for result in results:
+            score = 0.0
+            # 1. Semantic similarity score (if enabled)
+            if use_semantic:
+                semantic_score = await self._calculate_semantic_score(
+                    query, result.snippet + " " + result.title
+                )
+                score += semantic_score * self.semantic_weight
+            else:
+                # If semantic disabled, redistribute weight to keyword matching
+                score += result.relevance_score * (
+                    self.semantic_weight + self.keyword_weight
+                )
+            # 2. Keyword matching score (use existing relevance_score)
+            if not use_semantic:
+                # Already included above
+                pass
+            else:
+                score += result.relevance_score * self.keyword_weight
+            # 3. Metadata scoring (authority, content quality indicators)
+            metadata_score = self._calculate_metadata_score(result)
+            score += metadata_score * self.metadata_weight
+            # Store the hybrid score
+            result.relevance_score = score
+            scored_results.append(result)
+        # Sort by hybrid score
+        scored_results.sort(key=lambda r: r.relevance_score, reverse=True)
+        logger.debug(
+            f"Reranked results. Top score: {scored_results[0].relevance_score:.3f}"
+        )
+        return scored_results
+    async def _calculate_semantic_score(self, query: str, document: str) -> float:
+        """
+        Calculate semantic similarity between query and document.
+        Args:
+            query: Search query
+            document: Document text (title + snippet)
+        Returns:
+            Similarity score between 0 and 1
+        """
+        try:
+            # Generate embeddings
+            query_embedding = self.vector_engine.embed_documents([query])
+            doc_embedding = self.vector_engine.embed_documents([document])
+            # Calculate cosine similarity
+            import numpy as np
+            query_norm = query_embedding / np.linalg.norm(query_embedding)
+            doc_norm = doc_embedding / np.linalg.norm(doc_embedding)
+            similarity = np.dot(query_norm[0], doc_norm[0])
+            # Convert to 0-1 range (cosine similarity is -1 to 1)
+            score = (similarity + 1) / 2
+            return float(score)
+        except Exception as e:
+            logger.warning(f"Error calculating semantic score: {e}")
+            return 0.5  # Neutral score on error
+    def _calculate_metadata_score(self, result: SearchResult) -> float:
+        """
+        Calculate metadata-based score considering:
+        - Source authority (official docs > blogs > forums)
+        - Content type (tutorials/guides > reference > examples)
+        - Code examples presence
+        - Estimated quality indicators
+        Args:
+            result: Search result to score
+        Returns:
+            Metadata score between 0 and 1
+        """
+        score = 0.5  # Base score
+        # Source authority scoring
+        url_lower = result.url.lower()
+        if any(
+            domain in url_lower
+            for domain in [
+                "docs.python.org",
+                "fastapi.tiangolo.com",
+                "reactjs.org",
+                "docs.djangoproject.com",
+            ]
+        ):
+            score += 0.3  # Official documentation
+        elif any(
+            domain in url_lower
+            for domain in ["github.com", "readthedocs.io", "readthedocs.org"]
+        ):
+            score += 0.2  # Authoritative sources
+        elif any(
+            domain in url_lower
+            for domain in ["stackoverflow.com", "medium.com", "dev.to"]
+        ):
+            score += 0.1  # Community sources
+        # Content type scoring
+        content_type_scores = {
+            "tutorial": 0.2,
+            "guide": 0.2,
+            "reference": 0.15,
+            "example": 0.1,
+        }
+        score += content_type_scores.get(result.content_type.lower(), 0)
+        # Code examples boost
+        if result.code_snippets_count > 0:
+            score += 0.1
+        # URL structure quality (indicates well-organized docs)
+        if self._has_good_url_structure(result.url):
+            score += 0.05
+        # Normalize to 0-1 range
+        return min(1.0, max(0.0, score))
+    def _has_good_url_structure(self, url: str) -> bool:
+        """
+        Check if URL has good structure (versioned, organized).
+        Args:
+            url: URL to check
+        Returns:
+            True if URL has good structure
+        """
+        # Check for version in URL
+        has_version = bool(re.search(r"/v?\d+\.\d+/|/stable/|/latest/", url))
+        # Check for organized path structure
+        path_depth = len([p for p in url.split("/") if p]) - 2  # Exclude domain
+        has_good_depth = 2 <= path_depth <= 6
+        return has_version or has_good_depth
+# Global instance
+_reranker: Optional[SearchReranker] = None
+def get_reranker() -> SearchReranker:
+    """Get or create the global reranker instance."""
+    global _reranker
+    if _reranker is None:
+        _reranker = SearchReranker()
+    return _reranker