mcp-vector-search 0.12.6__py3-none-any.whl

mcp_vector_search/utils/gitignore.py

@@ -0,0 +1,250 @@
+"""Gitignore parsing and matching utilities."""
+
+import fnmatch
+import re
+from pathlib import Path
+
+from loguru import logger
+
+
+class GitignorePattern:
+    """Represents a single gitignore pattern with its matching logic."""
+
+    def __init__(
+        self, pattern: str, is_negation: bool = False, is_directory_only: bool = False
+    ):
+        """Initialize a gitignore pattern.
+
+        Args:
+            pattern: The pattern string
+            is_negation: Whether this is a negation pattern (starts with !)
+            is_directory_only: Whether this pattern only matches directories (ends with /)
+        """
+        self.original_pattern = pattern
+        self.is_negation = is_negation
+        self.is_directory_only = is_directory_only
+        self.pattern = self._normalize_pattern(pattern)
+
+    def _normalize_pattern(self, pattern: str) -> str:
+        """Normalize the pattern for matching."""
+        # Remove leading ! for negation patterns
+        if pattern.startswith("!"):
+            pattern = pattern[1:]
+
+        # Remove trailing / for directory-only patterns
+        if pattern.endswith("/"):
+            pattern = pattern[:-1]
+
+        # Handle leading slash (absolute from repo root)
+        if pattern.startswith("/"):
+            pattern = pattern[1:]
+
+        return pattern
+
+    def matches(self, path: str, is_directory: bool = False) -> bool:
+        """Check if this pattern matches the given path.
+
+        Args:
+            path: Relative path from repository root
+            is_directory: Whether the path is a directory
+
+        Returns:
+            True if the pattern matches
+        """
+        # Convert path separators for consistent matching
+        path = path.replace("\\", "/")
+        pattern = self.pattern.replace("\\", "/")
+
+        # For directory-only patterns, check if any parent directory matches
+        # This implements Git's behavior where "dir/" excludes both the directory
+        # AND all files within it recursively
+        if self.is_directory_only:
+            path_parts = path.split("/")
+            # Check each parent directory component
+            for i in range(1, len(path_parts) + 1):
+                parent = "/".join(path_parts[:i])
+                if fnmatch.fnmatch(parent, pattern):
+                    return True
+            # If no parent matches and this is not a directory, don't exclude
+            if not is_directory:
+                return False
+
+        # Try exact match first
+        if fnmatch.fnmatch(path, pattern):
+            return True
+
+        # Try matching any parent directory
+        path_parts = path.split("/")
+        for i in range(len(path_parts)):
+            subpath = "/".join(path_parts[i:])
+            if fnmatch.fnmatch(subpath, pattern):
+                return True
+
+        # Try matching with ** patterns (glob-style)
+        if "**" in pattern:
+            # Convert ** to regex pattern
+            regex_pattern = pattern.replace("**", ".*")
+            regex_pattern = regex_pattern.replace("*", "[^/]*")
+            regex_pattern = regex_pattern.replace("?", "[^/]")
+            regex_pattern = f"^{regex_pattern}$"
+
+            try:
+                if re.match(regex_pattern, path):
+                    return True
+            except re.error:
+                # Fallback to simple fnmatch if regex fails
+                pass
+
+        return False
+
+
+class GitignoreParser:
+    """Parser for .gitignore files with proper pattern matching."""
+
+    def __init__(self, project_root: Path):
+        """Initialize gitignore parser.
+
+        Args:
+            project_root: Root directory of the project
+        """
+        self.project_root = project_root
+        self.patterns: list[GitignorePattern] = []
+        self._load_gitignore_files()
+
+    def _load_gitignore_files(self) -> None:
+        """Load .gitignore file from project root only.
+
+        Note: Only the root .gitignore is loaded to avoid performance issues
+        with rglob traversing large directory trees (e.g., node_modules with
+        250K+ files). Subdirectory .gitignore files are intentionally skipped
+        as they would add significant overhead without much benefit for
+        semantic code search indexing.
+        """
+        # Load root .gitignore only
+        root_gitignore = self.project_root / ".gitignore"
+        if root_gitignore.exists():
+            self._parse_gitignore_file(root_gitignore)
+
+    def _parse_gitignore_file(self, gitignore_path: Path) -> None:
+        """Parse a single .gitignore file.
+
+        Args:
+            gitignore_path: Path to the .gitignore file
+        """
+        try:
+            with open(gitignore_path, encoding="utf-8", errors="ignore") as f:
+                lines = f.readlines()
+
+            for _line_num, line in enumerate(lines, 1):
+                line = line.strip()
+
+                # Skip empty lines and comments
+                if not line or line.startswith("#"):
+                    continue
+
+                # Check for negation pattern
+                is_negation = line.startswith("!")
+
+                # Check for directory-only pattern
+                is_directory_only = line.endswith("/")
+
+                # Create pattern (all patterns are from root .gitignore)
+                pattern = GitignorePattern(line, is_negation, is_directory_only)
+                self.patterns.append(pattern)
+
+        except Exception as e:
+            logger.warning(f"Failed to parse {gitignore_path}: {e}")
+
+    def is_ignored(self, path: Path, is_directory: bool | None = None) -> bool:
+        """Check if a path should be ignored according to .gitignore rules.
+
+        Args:
+            path: Path to check (can be absolute or relative to project root)
+            is_directory: Optional hint if path is a directory.
+                         If None, will check filesystem (slower).
+                         If provided, skips filesystem check (faster).
+
+        Returns:
+            True if the path should be ignored
+        """
+        try:
+            # SHORT-CIRCUIT: If no patterns, nothing is ignored
+            # This prevents 200k+ unnecessary filesystem stat() calls on projects
+            # without .gitignore files
+            if not self.patterns:
+                return False
+
+            # Convert to relative path from project root
+            if path.is_absolute():
+                relative_path = path.relative_to(self.project_root)
+            else:
+                relative_path = path
+
+            path_str = str(relative_path).replace("\\", "/")
+
+            # Only check if directory when needed and not provided as hint
+            # PERFORMANCE: Passing is_directory hint from caller (e.g., os.walk)
+            # avoids hundreds of thousands of stat() calls on large repositories
+            if is_directory is None:
+                is_directory = path.is_dir() if path.exists() else False
+
+            # Apply patterns in order, with later patterns overriding earlier ones
+            ignored = False
+
+            for pattern in self.patterns:
+                if pattern.matches(path_str, is_directory):
+                    ignored = not pattern.is_negation
+
+            return ignored
+
+        except ValueError:
+            # Path is not relative to project root
+            return False
+        except Exception as e:
+            logger.debug(f"Error checking gitignore for {path}: {e}")
+            return False
+
+    def get_ignored_patterns(self) -> list[str]:
+        """Get list of all ignore patterns.
+
+        Returns:
+            List of pattern strings
+        """
+        return [p.original_pattern for p in self.patterns if not p.is_negation]
+
+    def get_negation_patterns(self) -> list[str]:
+        """Get list of all negation patterns.
+
+        Returns:
+            List of negation pattern strings
+        """
+        return [p.original_pattern for p in self.patterns if p.is_negation]
+
+
+def create_gitignore_parser(project_root: Path) -> GitignoreParser:
+    """Create a gitignore parser for the given project.
+
+    Args:
+        project_root: Root directory of the project
+
+    Returns:
+        GitignoreParser instance
+    """
+    return GitignoreParser(project_root)
+
+
+def is_path_gitignored(
+    path: Path, project_root: Path, is_directory: bool | None = None
+) -> bool:
+    """Quick function to check if a path is gitignored.
+
+    Args:
+        path: Path to check
+        project_root: Root directory of the project
+        is_directory: Optional hint if path is a directory (avoids filesystem check)
+
+    Returns:
+        True if the path should be ignored
+    """
+    parser = create_gitignore_parser(project_root)
+    return parser.is_ignored(path, is_directory=is_directory)

mcp_vector_search/utils/monorepo.py

@@ -0,0 +1,277 @@
+"""Monorepo detection and subproject identification."""
+
+import json
+from pathlib import Path
+from typing import NamedTuple
+
+from loguru import logger
+
+
+class Subproject(NamedTuple):
+    """Represents a subproject in a monorepo."""
+
+    name: str  # "ewtn-plus-foundation"
+    path: Path  # Absolute path to subproject
+    relative_path: str  # Relative to monorepo root
+
+
+class MonorepoDetector:
+    """Detects monorepo structure and identifies subprojects."""
+
+    def __init__(self, project_root: Path):
+        """Initialize monorepo detector.
+
+        Args:
+            project_root: Root directory of the project
+        """
+        self.project_root = project_root
+        self._subprojects: list[Subproject] | None = None
+
+    def is_monorepo(self) -> bool:
+        """Check if project is a monorepo.
+
+        Returns:
+            True if monorepo structure detected
+        """
+        return bool(self.detect_subprojects())
+
+    def detect_subprojects(self) -> list[Subproject]:
+        """Detect all subprojects in the monorepo.
+
+        Returns:
+            List of detected subprojects
+        """
+        if self._subprojects is not None:
+            return self._subprojects
+
+        subprojects = []
+
+        # Try package.json workspaces (npm/yarn/pnpm)
+        subprojects.extend(self._detect_npm_workspaces())
+
+        # Try lerna.json
+        if not subprojects:
+            subprojects.extend(self._detect_lerna_packages())
+
+        # Try pnpm-workspace.yaml
+        if not subprojects:
+            subprojects.extend(self._detect_pnpm_workspaces())
+
+        # Try nx workspace
+        if not subprojects:
+            subprojects.extend(self._detect_nx_workspace())
+
+        # Fallback: Look for multiple package.json files
+        if not subprojects:
+            subprojects.extend(self._detect_by_package_json())
+
+        self._subprojects = subprojects
+        logger.debug(f"Detected {len(subprojects)} subprojects in {self.project_root}")
+
+        return subprojects
+
+    def _detect_npm_workspaces(self) -> list[Subproject]:
+        """Detect npm/yarn/pnpm workspaces from package.json.
+
+        Returns:
+            List of subprojects from workspaces
+        """
+        package_json = self.project_root / "package.json"
+        if not package_json.exists():
+            return []
+
+        try:
+            with open(package_json) as f:
+                data = json.load(f)
+
+            workspaces = data.get("workspaces", [])
+
+            # Handle both array and object format
+            if isinstance(workspaces, dict):
+                workspaces = workspaces.get("packages", [])
+
+            return self._expand_workspace_patterns(workspaces)
+
+        except Exception as e:
+            logger.debug(f"Failed to parse package.json workspaces: {e}")
+            return []
+
+    def _detect_lerna_packages(self) -> list[Subproject]:
+        """Detect lerna packages from lerna.json.
+
+        Returns:
+            List of subprojects from lerna
+        """
+        lerna_json = self.project_root / "lerna.json"
+        if not lerna_json.exists():
+            return []
+
+        try:
+            with open(lerna_json) as f:
+                data = json.load(f)
+
+            packages = data.get("packages", ["packages/*"])
+            return self._expand_workspace_patterns(packages)
+
+        except Exception as e:
+            logger.debug(f"Failed to parse lerna.json: {e}")
+            return []
+
+    def _detect_pnpm_workspaces(self) -> list[Subproject]:
+        """Detect pnpm workspaces from pnpm-workspace.yaml.
+
+        Returns:
+            List of subprojects from pnpm
+        """
+        pnpm_workspace = self.project_root / "pnpm-workspace.yaml"
+        if not pnpm_workspace.exists():
+            return []
+
+        try:
+            import yaml
+
+            with open(pnpm_workspace) as f:
+                data = yaml.safe_load(f)
+
+            packages = data.get("packages", [])
+            return self._expand_workspace_patterns(packages)
+
+        except ImportError:
+            logger.debug("pyyaml not installed, skipping pnpm-workspace.yaml detection")
+            return []
+        except Exception as e:
+            logger.debug(f"Failed to parse pnpm-workspace.yaml: {e}")
+            return []
+
+    def _detect_nx_workspace(self) -> list[Subproject]:
+        """Detect nx workspace projects.
+
+        Returns:
+            List of subprojects from nx workspace
+        """
+        nx_json = self.project_root / "nx.json"
+        workspace_json = self.project_root / "workspace.json"
+
+        if not (nx_json.exists() or workspace_json.exists()):
+            return []
+
+        # Nx projects are typically in apps/ and libs/
+        subprojects = []
+        for base_dir in ["apps", "libs", "packages"]:
+            base_path = self.project_root / base_dir
+            if base_path.exists():
+                for subdir in base_path.iterdir():
+                    if subdir.is_dir() and not subdir.name.startswith("."):
+                        package_json = subdir / "package.json"
+                        name = self._get_package_name(package_json) or subdir.name
+                        relative = str(subdir.relative_to(self.project_root))
+                        subprojects.append(Subproject(name, subdir, relative))
+
+        return subprojects
+
+    def _detect_by_package_json(self) -> list[Subproject]:
+        """Fallback: Find all directories with package.json.
+
+        Returns:
+            List of subprojects by package.json presence
+        """
+        subprojects = []
+
+        # Only search up to 3 levels deep
+        for package_json in self.project_root.rglob("package.json"):
+            # Skip node_modules
+            if "node_modules" in package_json.parts:
+                continue
+
+            # Skip root package.json
+            if package_json.parent == self.project_root:
+                continue
+
+            # Check depth
+            relative_parts = package_json.relative_to(self.project_root).parts
+            if len(relative_parts) > 4:  # Too deep
+                continue
+
+            subdir = package_json.parent
+            name = self._get_package_name(package_json) or subdir.name
+            relative = str(subdir.relative_to(self.project_root))
+            subprojects.append(Subproject(name, subdir, relative))
+
+        return subprojects
+
+    def _expand_workspace_patterns(self, patterns: list[str]) -> list[Subproject]:
+        """Expand workspace glob patterns to actual directories.
+
+        Args:
+            patterns: List of glob patterns (e.g., ["packages/*", "apps/*"])
+
+        Returns:
+            List of subprojects matching patterns
+        """
+        subprojects = []
+
+        for pattern in patterns:
+            # Remove negation patterns (e.g., "!packages/excluded")
+            if pattern.startswith("!"):
+                continue
+
+            # Expand glob pattern
+            for path in self.project_root.glob(pattern):
+                if not path.is_dir():
+                    continue
+
+                if path.name.startswith("."):
+                    continue
+
+                # Try to get name from package.json
+                package_json = path / "package.json"
+                name = self._get_package_name(package_json) or path.name
+                relative = str(path.relative_to(self.project_root))
+
+                subprojects.append(Subproject(name, path, relative))
+
+        return subprojects
+
+    def _get_package_name(self, package_json: Path) -> str | None:
+        """Get package name from package.json.
+
+        Args:
+            package_json: Path to package.json file
+
+        Returns:
+            Package name or None
+        """
+        if not package_json.exists():
+            return None
+
+        try:
+            with open(package_json) as f:
+                data = json.load(f)
+            return data.get("name")
+        except Exception:
+            return None
+
+    def get_subproject_for_file(self, file_path: Path) -> Subproject | None:
+        """Determine which subproject a file belongs to.
+
+        Args:
+            file_path: Path to file
+
+        Returns:
+            Subproject containing the file, or None
+        """
+        subprojects = self.detect_subprojects()
+
+        if not subprojects:
+            return None
+
+        # Find the most specific (deepest) subproject containing this file
+        matching_subprojects = [
+            sp for sp in subprojects if file_path.is_relative_to(sp.path)
+        ]
+
+        if not matching_subprojects:
+            return None
+
+        # Return the deepest match (longest path)
+        return max(matching_subprojects, key=lambda sp: len(sp.path.parts))