mcp-vector-search 0.15.7__py3-none-any.whl

mcp_vector_search/utils/monorepo.py

@@ -0,0 +1,339 @@
+"""Monorepo detection and subproject identification."""
+
+import json
+from pathlib import Path
+from typing import NamedTuple
+
+from loguru import logger
+
+# Directories to exclude from subproject detection
+# These are typically test/example/docs directories, not actual subprojects
+EXCLUDED_SUBPROJECT_DIRS = {
+    "tests",
+    "test",
+    "examples",
+    "example",
+    "docs",
+    "doc",
+    "scripts",
+    "tools",
+    "benchmarks",
+    "benchmark",
+    "node_modules",
+    ".git",
+    ".github",
+    ".gitlab",
+    "build",
+    "dist",
+    "__pycache__",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    "coverage",
+    ".coverage",
+    "htmlcov",
+}
+
+
+class Subproject(NamedTuple):
+    """Represents a subproject in a monorepo."""
+
+    name: str  # "ewtn-plus-foundation"
+    path: Path  # Absolute path to subproject
+    relative_path: str  # Relative to monorepo root
+
+
+class MonorepoDetector:
+    """Detects monorepo structure and identifies subprojects."""
+
+    def __init__(self, project_root: Path):
+        """Initialize monorepo detector.
+
+        Args:
+            project_root: Root directory of the project
+        """
+        self.project_root = project_root
+        self._subprojects: list[Subproject] | None = None
+
+    def _is_excluded_path(self, path: Path) -> bool:
+        """Check if a path should be excluded from subproject detection.
+
+        Args:
+            path: Path to check (relative to project root)
+
+        Returns:
+            True if path should be excluded from subproject detection
+        """
+        try:
+            relative_path = path.relative_to(self.project_root)
+            # Check if any part of the path is in the excluded set
+            return any(part in EXCLUDED_SUBPROJECT_DIRS for part in relative_path.parts)
+        except ValueError:
+            # Path is not relative to project root
+            return True
+
+    def is_monorepo(self) -> bool:
+        """Check if project is a monorepo.
+
+        Returns:
+            True if monorepo structure detected
+        """
+        return bool(self.detect_subprojects())
+
+    def detect_subprojects(self) -> list[Subproject]:
+        """Detect all subprojects in the monorepo.
+
+        Returns:
+            List of detected subprojects
+        """
+        if self._subprojects is not None:
+            return self._subprojects
+
+        subprojects = []
+
+        # Try package.json workspaces (npm/yarn/pnpm)
+        subprojects.extend(self._detect_npm_workspaces())
+
+        # Try lerna.json
+        if not subprojects:
+            subprojects.extend(self._detect_lerna_packages())
+
+        # Try pnpm-workspace.yaml
+        if not subprojects:
+            subprojects.extend(self._detect_pnpm_workspaces())
+
+        # Try nx workspace
+        if not subprojects:
+            subprojects.extend(self._detect_nx_workspace())
+
+        # Fallback: Look for multiple package.json files
+        if not subprojects:
+            subprojects.extend(self._detect_by_package_json())
+
+        self._subprojects = subprojects
+        logger.debug(f"Detected {len(subprojects)} subprojects in {self.project_root}")
+
+        return subprojects
+
+    def _detect_npm_workspaces(self) -> list[Subproject]:
+        """Detect npm/yarn/pnpm workspaces from package.json.
+
+        Returns:
+            List of subprojects from workspaces
+        """
+        package_json = self.project_root / "package.json"
+        if not package_json.exists():
+            return []
+
+        try:
+            with open(package_json) as f:
+                data = json.load(f)
+
+            workspaces = data.get("workspaces", [])
+
+            # Handle both array and object format
+            if isinstance(workspaces, dict):
+                workspaces = workspaces.get("packages", [])
+
+            return self._expand_workspace_patterns(workspaces)
+
+        except Exception as e:
+            logger.debug(f"Failed to parse package.json workspaces: {e}")
+            return []
+
+    def _detect_lerna_packages(self) -> list[Subproject]:
+        """Detect lerna packages from lerna.json.
+
+        Returns:
+            List of subprojects from lerna
+        """
+        lerna_json = self.project_root / "lerna.json"
+        if not lerna_json.exists():
+            return []
+
+        try:
+            with open(lerna_json) as f:
+                data = json.load(f)
+
+            packages = data.get("packages", ["packages/*"])
+            return self._expand_workspace_patterns(packages)
+
+        except Exception as e:
+            logger.debug(f"Failed to parse lerna.json: {e}")
+            return []
+
+    def _detect_pnpm_workspaces(self) -> list[Subproject]:
+        """Detect pnpm workspaces from pnpm-workspace.yaml.
+
+        Returns:
+            List of subprojects from pnpm
+        """
+        pnpm_workspace = self.project_root / "pnpm-workspace.yaml"
+        if not pnpm_workspace.exists():
+            return []
+
+        try:
+            import yaml
+
+            with open(pnpm_workspace) as f:
+                data = yaml.safe_load(f)
+
+            packages = data.get("packages", [])
+            return self._expand_workspace_patterns(packages)
+
+        except ImportError:
+            logger.debug("pyyaml not installed, skipping pnpm-workspace.yaml detection")
+            return []
+        except Exception as e:
+            logger.debug(f"Failed to parse pnpm-workspace.yaml: {e}")
+            return []
+
+    def _detect_nx_workspace(self) -> list[Subproject]:
+        """Detect nx workspace projects.
+
+        Returns:
+            List of subprojects from nx workspace
+        """
+        nx_json = self.project_root / "nx.json"
+        workspace_json = self.project_root / "workspace.json"
+
+        if not (nx_json.exists() or workspace_json.exists()):
+            return []
+
+        # Nx projects are typically in apps/ and libs/
+        subprojects = []
+        for base_dir in ["apps", "libs", "packages"]:
+            base_path = self.project_root / base_dir
+            if base_path.exists():
+                for subdir in base_path.iterdir():
+                    if subdir.is_dir() and not subdir.name.startswith("."):
+                        # Skip excluded directories
+                        if self._is_excluded_path(subdir):
+                            logger.debug(
+                                f"Skipping excluded nx workspace path: {subdir.relative_to(self.project_root)}"
+                            )
+                            continue
+
+                        package_json = subdir / "package.json"
+                        name = self._get_package_name(package_json) or subdir.name
+                        relative = str(subdir.relative_to(self.project_root))
+                        subprojects.append(Subproject(name, subdir, relative))
+
+        return subprojects
+
+    def _detect_by_package_json(self) -> list[Subproject]:
+        """Fallback: Find all directories with package.json.
+
+        Returns:
+            List of subprojects by package.json presence
+        """
+        subprojects = []
+
+        # Only search up to 3 levels deep
+        for package_json in self.project_root.rglob("package.json"):
+            # Skip root package.json
+            if package_json.parent == self.project_root:
+                continue
+
+            # Skip excluded directories (tests, examples, docs, etc.)
+            if self._is_excluded_path(package_json.parent):
+                logger.debug(
+                    f"Skipping excluded path: {package_json.relative_to(self.project_root)}"
+                )
+                continue
+
+            # Check depth
+            relative_parts = package_json.relative_to(self.project_root).parts
+            if len(relative_parts) > 4:  # Too deep
+                continue
+
+            subdir = package_json.parent
+            name = self._get_package_name(package_json) or subdir.name
+            relative = str(subdir.relative_to(self.project_root))
+            subprojects.append(Subproject(name, subdir, relative))
+
+        return subprojects
+
+    def _expand_workspace_patterns(self, patterns: list[str]) -> list[Subproject]:
+        """Expand workspace glob patterns to actual directories.
+
+        Args:
+            patterns: List of glob patterns (e.g., ["packages/*", "apps/*"])
+
+        Returns:
+            List of subprojects matching patterns
+        """
+        subprojects = []
+
+        for pattern in patterns:
+            # Remove negation patterns (e.g., "!packages/excluded")
+            if pattern.startswith("!"):
+                continue
+
+            # Expand glob pattern
+            for path in self.project_root.glob(pattern):
+                if not path.is_dir():
+                    continue
+
+                if path.name.startswith("."):
+                    continue
+
+                # Skip excluded directories (tests, examples, docs, etc.)
+                if self._is_excluded_path(path):
+                    logger.debug(
+                        f"Skipping excluded workspace path: {path.relative_to(self.project_root)}"
+                    )
+                    continue
+
+                # Try to get name from package.json
+                package_json = path / "package.json"
+                name = self._get_package_name(package_json) or path.name
+                relative = str(path.relative_to(self.project_root))
+
+                subprojects.append(Subproject(name, path, relative))
+
+        return subprojects
+
+    def _get_package_name(self, package_json: Path) -> str | None:
+        """Get package name from package.json.
+
+        Args:
+            package_json: Path to package.json file
+
+        Returns:
+            Package name or None
+        """
+        if not package_json.exists():
+            return None
+
+        try:
+            with open(package_json) as f:
+                data = json.load(f)
+            return data.get("name")
+        except Exception:
+            return None
+
+    def get_subproject_for_file(self, file_path: Path) -> Subproject | None:
+        """Determine which subproject a file belongs to.
+
+        Args:
+            file_path: Path to file
+
+        Returns:
+            Subproject containing the file, or None
+        """
+        subprojects = self.detect_subprojects()
+
+        if not subprojects:
+            return None
+
+        # Find the most specific (deepest) subproject containing this file
+        matching_subprojects = [
+            sp for sp in subprojects if file_path.is_relative_to(sp.path)
+        ]
+
+        if not matching_subprojects:
+            return None
+
+        # Return the deepest match (longest path)
+        return max(matching_subprojects, key=lambda sp: len(sp.path.parts))

mcp_vector_search/utils/timing.py

@@ -0,0 +1,338 @@
+"""Timing utilities for performance measurement and optimization."""
+
+import asyncio
+import json
+import statistics
+import time
+from collections.abc import Callable
+from contextlib import asynccontextmanager, contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+
+@dataclass
+class TimingResult:
+    """Result of a timing measurement."""
+
+    operation: str
+    duration: float  # in seconds
+    timestamp: float
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def duration_ms(self) -> float:
+        """Duration in milliseconds."""
+        return self.duration * 1000
+
+    @property
+    def duration_us(self) -> float:
+        """Duration in microseconds."""
+        return self.duration * 1_000_000
+
+
+class PerformanceProfiler:
+    """Performance profiler for measuring and analyzing operation timings."""
+
+    def __init__(self, name: str = "default"):
+        self.name = name
+        self.results: list[TimingResult] = []
+        self._active_timers: dict[str, float] = {}
+        self._nested_level = 0
+
+    def start_timer(self, operation: str) -> None:
+        """Start timing an operation."""
+        if operation in self._active_timers:
+            logger.warning(f"Timer '{operation}' already active, overwriting")
+        self._active_timers[operation] = time.perf_counter()
+
+    def stop_timer(
+        self, operation: str, metadata: dict[str, Any] | None = None
+    ) -> TimingResult:
+        """Stop timing an operation and record the result."""
+        if operation not in self._active_timers:
+            raise ValueError(f"Timer '{operation}' not found or not started")
+
+        start_time = self._active_timers.pop(operation)
+        duration = time.perf_counter() - start_time
+
+        result = TimingResult(
+            operation=operation,
+            duration=duration,
+            timestamp=time.time(),
+            metadata=metadata or {},
+        )
+
+        self.results.append(result)
+        return result
+
+    @contextmanager
+    def time_operation(self, operation: str, metadata: dict[str, Any] | None = None):
+        """Context manager for timing an operation."""
+        indent = "  " * self._nested_level
+        logger.debug(f"{indent}⏱️  Starting: {operation}")
+
+        self._nested_level += 1
+        start_time = time.perf_counter()
+
+        try:
+            yield
+        finally:
+            duration = time.perf_counter() - start_time
+            self._nested_level -= 1
+
+            result = TimingResult(
+                operation=operation,
+                duration=duration,
+                timestamp=time.time(),
+                metadata=metadata or {},
+            )
+
+            self.results.append(result)
+
+            indent = "  " * self._nested_level
+            logger.debug(f"{indent}✅ Completed: {operation} ({duration * 1000:.2f}ms)")
+
+    @asynccontextmanager
+    async def time_async_operation(
+        self, operation: str, metadata: dict[str, Any] | None = None
+    ):
+        """Async context manager for timing an operation."""
+        indent = "  " * self._nested_level
+        logger.debug(f"{indent}⏱️  Starting: {operation}")
+
+        self._nested_level += 1
+        start_time = time.perf_counter()
+
+        try:
+            yield
+        finally:
+            duration = time.perf_counter() - start_time
+            self._nested_level -= 1
+
+            result = TimingResult(
+                operation=operation,
+                duration=duration,
+                timestamp=time.time(),
+                metadata=metadata or {},
+            )
+
+            self.results.append(result)
+
+            indent = "  " * self._nested_level
+            logger.debug(f"{indent}✅ Completed: {operation} ({duration * 1000:.2f}ms)")
+
+    def get_stats(self, operation: str | None = None) -> dict[str, Any]:
+        """Get timing statistics for operations."""
+        if operation:
+            durations = [r.duration for r in self.results if r.operation == operation]
+        else:
+            durations = [r.duration for r in self.results]
+
+        if not durations:
+            return {}
+
+        return {
+            "count": len(durations),
+            "total": sum(durations),
+            "mean": statistics.mean(durations),
+            "median": statistics.median(durations),
+            "min": min(durations),
+            "max": max(durations),
+            "std_dev": statistics.stdev(durations) if len(durations) > 1 else 0.0,
+            "p95": (
+                statistics.quantiles(durations, n=20)[18]
+                if len(durations) >= 20
+                else max(durations)
+            ),
+            "p99": (
+                statistics.quantiles(durations, n=100)[98]
+                if len(durations) >= 100
+                else max(durations)
+            ),
+        }
+
+    def get_operation_breakdown(self) -> dict[str, dict[str, Any]]:
+        """Get breakdown of all operations."""
+        operations = {r.operation for r in self.results}
+        return {op: self.get_stats(op) for op in operations}
+
+    def print_report(self, show_individual: bool = False, min_duration_ms: float = 0.0):
+        """Print a detailed performance report."""
+        if not self.results:
+            print("No timing results recorded.")
+            return
+
+        print(f"\n{'=' * 60}")
+        print(f"PERFORMANCE REPORT: {self.name}")
+        print(f"{'=' * 60}")
+
+        # Overall stats
+        overall_stats = self.get_stats()
+        print("\nOVERALL STATISTICS:")
+        print(f"  Total operations: {overall_stats['count']}")
+        print(f"  Total time: {overall_stats['total'] * 1000:.2f}ms")
+        print(f"  Average: {overall_stats['mean'] * 1000:.2f}ms")
+        print(f"  Median: {overall_stats['median'] * 1000:.2f}ms")
+        print(f"  Min: {overall_stats['min'] * 1000:.2f}ms")
+        print(f"  Max: {overall_stats['max'] * 1000:.2f}ms")
+
+        # Per-operation breakdown
+        breakdown = self.get_operation_breakdown()
+        print("\nPER-OPERATION BREAKDOWN:")
+
+        for operation, stats in sorted(
+            breakdown.items(), key=lambda x: x[1]["total"], reverse=True
+        ):
+            print(f"\n  {operation}:")
+            print(f"    Count: {stats['count']}")
+            print(
+                f"    Total: {stats['total'] * 1000:.2f}ms ({stats['total'] / overall_stats['total'] * 100:.1f}%)"
+            )
+            print(f"    Average: {stats['mean'] * 1000:.2f}ms")
+            print(
+                f"    Min/Max: {stats['min'] * 1000:.2f}ms / {stats['max'] * 1000:.2f}ms"
+            )
+            if stats["count"] > 1:
+                print(f"    StdDev: {stats['std_dev'] * 1000:.2f}ms")
+
+        # Individual results if requested
+        if show_individual:
+            print("\nINDIVIDUAL RESULTS:")
+            for result in self.results:
+                if result.duration_ms >= min_duration_ms:
+                    print(f"  {result.operation}: {result.duration_ms:.2f}ms")
+                    if result.metadata:
+                        print(f"    Metadata: {result.metadata}")
+
+    def save_results(self, file_path: Path):
+        """Save timing results to a JSON file."""
+        data = {
+            "profiler_name": self.name,
+            "timestamp": time.time(),
+            "results": [
+                {
+                    "operation": r.operation,
+                    "duration": r.duration,
+                    "timestamp": r.timestamp,
+                    "metadata": r.metadata,
+                }
+                for r in self.results
+            ],
+            "stats": self.get_operation_breakdown(),
+        }
+
+        with open(file_path, "w") as f:
+            json.dump(data, f, indent=2)
+
+    def clear(self):
+        """Clear all timing results."""
+        self.results.clear()
+        self._active_timers.clear()
+        self._nested_level = 0
+
+
+# Global profiler instance
+_global_profiler = PerformanceProfiler("global")
+
+
+def time_function(
+    operation_name: str | None = None, metadata: dict[str, Any] | None = None
+):
+    """Decorator for timing function execution."""
+
+    def decorator(func: Callable) -> Callable:
+        name = operation_name or f"{func.__module__}.{func.__name__}"
+
+        if asyncio.iscoroutinefunction(func):
+
+            async def async_wrapper(*args, **kwargs):
+                async with _global_profiler.time_async_operation(name, metadata):
+                    return await func(*args, **kwargs)
+
+            return async_wrapper
+        else:
+
+            def sync_wrapper(*args, **kwargs):
+                with _global_profiler.time_operation(name, metadata):
+                    return func(*args, **kwargs)
+
+            return sync_wrapper
+
+    return decorator
+
+
+@contextmanager
+def time_block(operation: str, metadata: dict[str, Any] | None = None):
+    """Context manager for timing a block of code using the global profiler."""
+    with _global_profiler.time_operation(operation, metadata):
+        yield
+
+
+@asynccontextmanager
+async def time_async_block(operation: str, metadata: dict[str, Any] | None = None):
+    """Async context manager for timing a block of code using the global profiler."""
+    async with _global_profiler.time_async_operation(operation, metadata):
+        yield
+
+
+def get_global_profiler() -> PerformanceProfiler:
+    """Get the global profiler instance."""
+    return _global_profiler
+
+
+def print_global_report(**kwargs):
+    """Print report from the global profiler."""
+    _global_profiler.print_report(**kwargs)
+
+
+def clear_global_profiler():
+    """Clear the global profiler."""
+    _global_profiler.clear()
+
+
+class SearchProfiler(PerformanceProfiler):
+    """Specialized profiler for search operations."""
+
+    def __init__(self):
+        super().__init__("search_profiler")
+
+    async def profile_search(
+        self, search_func: Callable, query: str, **search_kwargs
+    ) -> tuple[Any, dict[str, float]]:
+        """Profile a complete search operation with detailed breakdown."""
+
+        async with self.time_async_operation(
+            "total_search", {"query": query, "kwargs": search_kwargs}
+        ):
+            # Time the actual search
+            async with self.time_async_operation("search_execution", {"query": query}):
+                result = await search_func(query, **search_kwargs)
+
+            # Time result processing if we can measure it
+            async with self.time_async_operation(
+                "result_processing",
+                {"result_count": len(result) if hasattr(result, "__len__") else 0},
+            ):
+                # Simulate any post-processing that might happen
+                await asyncio.sleep(0)  # Placeholder for actual processing
+
+        # Return results and timing breakdown
+        timing_breakdown = {
+            op: self.get_stats(op)["mean"] * 1000  # Convert to ms
+            for op in ["total_search", "search_execution", "result_processing"]
+            if self.get_stats(op)
+        }
+
+        return result, timing_breakdown
+
+
+# Convenience function for quick search profiling
+async def profile_search_operation(
+    search_func: Callable, query: str, **kwargs
+) -> tuple[Any, dict[str, float]]:
+    """Quick function to profile a search operation."""
+    profiler = SearchProfiler()
+    return await profiler.profile_search(search_func, query, **kwargs)