gobby 0.2.5__py3-none-any.whl

gobby/mcp_proxy/services/fallback.py

@@ -0,0 +1,306 @@
+"""Tool fallback resolution service.
+
+Provides alternative tool suggestions when a tool call fails,
+using semantic similarity and success rate weighting.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from gobby.mcp_proxy.metrics import ToolMetricsManager
+    from gobby.mcp_proxy.semantic_search import SemanticToolSearch
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FallbackSuggestion:
+    """A suggested alternative tool."""
+
+    server_name: str
+    tool_name: str
+    description: str | None
+    similarity: float
+    success_rate: float | None
+    score: float  # Combined ranking score
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "server_name": self.server_name,
+            "tool_name": self.tool_name,
+            "description": self.description,
+            "similarity": round(self.similarity, 4),
+            "success_rate": round(self.success_rate, 4) if self.success_rate else None,
+            "score": round(self.score, 4),
+        }
+
+
+class ToolFallbackResolver:
+    """
+    Resolves alternative tools when a tool call fails.
+
+    Uses semantic similarity search to find similar tools and
+    weights results by historical success rate from metrics.
+    """
+
+    # Default weight for similarity vs success_rate in scoring
+    DEFAULT_SIMILARITY_WEIGHT = 0.7
+    DEFAULT_SUCCESS_WEIGHT = 0.3
+
+    # Minimum similarity threshold for candidates
+    DEFAULT_MIN_SIMILARITY = 0.3
+
+    # Default success rate when no metrics available
+    DEFAULT_SUCCESS_RATE = 0.5
+
+    def __init__(
+        self,
+        semantic_search: SemanticToolSearch,
+        metrics_manager: ToolMetricsManager | None = None,
+        similarity_weight: float = DEFAULT_SIMILARITY_WEIGHT,
+        success_weight: float = DEFAULT_SUCCESS_WEIGHT,
+        min_similarity: float = DEFAULT_MIN_SIMILARITY,
+    ):
+        """
+        Initialize the fallback resolver.
+
+        Args:
+            semantic_search: SemanticToolSearch instance for finding similar tools
+            metrics_manager: Optional ToolMetricsManager for success rate data
+            similarity_weight: Weight for similarity score (0-1)
+            success_weight: Weight for success rate score (0-1)
+            min_similarity: Minimum similarity threshold for candidates
+        """
+        self._semantic_search = semantic_search
+        self._metrics_manager = metrics_manager
+        self._similarity_weight = similarity_weight
+        self._success_weight = success_weight
+        self._min_similarity = min_similarity
+
+    async def find_alternatives(
+        self,
+        failed_tool_name: str,
+        failed_tool_description: str | None = None,
+        error_context: str | None = None,
+        server_name: str | None = None,
+        project_id: str | None = None,
+        top_k: int = 5,
+        exclude_failed: bool = True,
+    ) -> list[FallbackSuggestion]:
+        """
+        Find alternative tools similar to a failed tool.
+
+        Uses semantic search to find tools with similar descriptions,
+        then weights by historical success rate.
+
+        Args:
+            failed_tool_name: Name of the tool that failed
+            failed_tool_description: Description of the failed tool (if available)
+            error_context: Error message or context for better matching
+            server_name: Server the failed tool belongs to (for exclusion)
+            project_id: Project ID for scoping the search
+            top_k: Maximum number of suggestions to return
+            exclude_failed: Whether to exclude the failed tool from results
+
+        Returns:
+            List of FallbackSuggestion sorted by combined score (descending)
+        """
+        if not project_id:
+            logger.warning("No project_id provided for fallback search")
+            return []
+
+        # Build query from tool info and error context
+        query = self._build_search_query(failed_tool_name, failed_tool_description, error_context)
+
+        # Get semantically similar tools
+        try:
+            search_results = await self._semantic_search.search_tools(
+                query=query,
+                project_id=project_id,
+                top_k=top_k * 2,  # Get extra for filtering
+                min_similarity=self._min_similarity,
+            )
+        except Exception as e:
+            logger.error(f"Semantic search failed in fallback resolver: {e}")
+            return []
+
+        if not search_results:
+            logger.debug(f"No semantic matches found for '{failed_tool_name}'")
+            return []
+
+        # Filter out the failed tool if requested
+        if exclude_failed:
+            search_results = [
+                r
+                for r in search_results
+                if not (r.tool_name == failed_tool_name and r.server_name == server_name)
+            ]
+
+        # Enrich with success rates and compute combined scores
+        suggestions = []
+        for result in search_results[:top_k]:
+            success_rate = self._get_success_rate(result.server_name, result.tool_name, project_id)
+
+            score = self._compute_score(result.similarity, success_rate)
+
+            suggestions.append(
+                FallbackSuggestion(
+                    server_name=result.server_name,
+                    tool_name=result.tool_name,
+                    description=result.description,
+                    similarity=result.similarity,
+                    success_rate=success_rate,
+                    score=score,
+                )
+            )
+
+        # Sort by combined score (descending)
+        suggestions.sort(key=lambda s: s.score, reverse=True)
+
+        logger.debug(f"Found {len(suggestions)} fallback suggestions for '{failed_tool_name}'")
+        return suggestions
+
+    def _build_search_query(
+        self,
+        tool_name: str,
+        description: str | None,
+        error_context: str | None,
+    ) -> str:
+        """
+        Build a search query from tool info and error context.
+
+        Args:
+            tool_name: Name of the failed tool
+            description: Tool description
+            error_context: Error message or context
+
+        Returns:
+            Search query string
+        """
+        parts = [f"Tool similar to: {tool_name}"]
+
+        if description:
+            parts.append(f"Description: {description}")
+
+        if error_context:
+            # Extract key terms from error, avoiding noise
+            parts.append(f"Context: {error_context[:200]}")
+
+        return "\n".join(parts)
+
+    def _get_success_rate(self, server_name: str, tool_name: str, project_id: str) -> float | None:
+        """
+        Get success rate for a tool from metrics.
+
+        Args:
+            server_name: Server name
+            tool_name: Tool name
+            project_id: Project ID
+
+        Returns:
+            Success rate (0-1) or None if no metrics available
+        """
+        if not self._metrics_manager:
+            return None
+
+        try:
+            return self._metrics_manager.get_tool_success_rate(
+                server_name=server_name,
+                tool_name=tool_name,
+                project_id=project_id,
+            )
+        except Exception as e:
+            logger.debug(f"Failed to get success rate for {server_name}/{tool_name}: {e}")
+            return None
+
+    def _compute_score(self, similarity: float, success_rate: float | None) -> float:
+        """
+        Compute combined ranking score.
+
+        Score = (similarity * similarity_weight) + (success_rate * success_weight)
+
+        When success_rate is None, uses default value to avoid penalizing
+        tools without metrics history.
+
+        Args:
+            similarity: Cosine similarity score (0-1)
+            success_rate: Historical success rate (0-1) or None
+
+        Returns:
+            Combined score (0-1)
+        """
+        effective_success_rate = (
+            success_rate if success_rate is not None else self.DEFAULT_SUCCESS_RATE
+        )
+
+        return similarity * self._similarity_weight + effective_success_rate * self._success_weight
+
+    async def find_alternatives_for_error(
+        self,
+        server_name: str,
+        tool_name: str,
+        error_message: str,
+        project_id: str,
+        top_k: int = 3,
+    ) -> list[dict[str, Any]]:
+        """
+        Convenience method for call_tool integration.
+
+        Takes error details and returns serialized suggestions.
+
+        Args:
+            server_name: Server where the tool failed
+            tool_name: Name of the failed tool
+            error_message: Error message from the failure
+            project_id: Project ID
+            top_k: Number of suggestions to return
+
+        Returns:
+            List of suggestion dictionaries ready for JSON response
+        """
+        # Try to get tool description from cached tools
+        description = await self._get_tool_description(server_name, tool_name)
+
+        suggestions = await self.find_alternatives(
+            failed_tool_name=tool_name,
+            failed_tool_description=description,
+            error_context=error_message,
+            server_name=server_name,
+            project_id=project_id,
+            top_k=top_k,
+        )
+
+        return [s.to_dict() for s in suggestions]
+
+    async def _get_tool_description(self, server_name: str, tool_name: str) -> str | None:
+        """
+        Get tool description from semantic search's cached data.
+
+        Args:
+            server_name: Server name
+            tool_name: Tool name
+
+        Returns:
+            Tool description or None
+        """
+        # The tool info is in the database, accessed via _get_tool_info_map
+        # But we don't have project_id here, so we search all
+        try:
+            row = self._semantic_search.db.fetchone(
+                """
+                SELECT t.description
+                FROM tools t
+                JOIN mcp_servers s ON t.mcp_server_id = s.id
+                WHERE s.name = ? AND t.name = ?
+                LIMIT 1
+                """,
+                (server_name, tool_name),
+            )
+            return row["description"] if row else None
+        except Exception:
+            return None

gobby/mcp_proxy/services/recommendation.py

@@ -0,0 +1,224 @@
+"""Recommendation service."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+    from gobby.config.app import RecommendToolsConfig
+
+logger = logging.getLogger("gobby.mcp.server")
+
+# Search mode type
+SearchMode = Literal["llm", "semantic", "hybrid"]
+
+
+class RecommendationService:
+    """Service for recommending tools."""
+
+    def __init__(
+        self,
+        llm_service: Any,
+        mcp_manager: Any,
+        semantic_search: Any | None = None,
+        project_id: str | None = None,
+        config: RecommendToolsConfig | None = None,
+    ):
+        self._llm_service = llm_service
+        self._mcp_manager = mcp_manager
+        self._semantic_search = semantic_search
+        self._project_id = project_id
+        self._config = config
+
+    def _get_config(self) -> RecommendToolsConfig:
+        """Get config with fallback to defaults."""
+        if self._config is not None:
+            return self._config
+        from gobby.config.app import RecommendToolsConfig
+
+        return RecommendToolsConfig()
+
+    async def recommend_tools(
+        self,
+        task_description: str,
+        agent_id: str | None = None,
+        search_mode: SearchMode = "llm",
+        top_k: int = 10,
+        min_similarity: float = 0.3,
+        project_id: str | None = None,
+    ) -> dict[str, Any]:
+        """
+        Recommend tools based on task description.
+
+        Args:
+            task_description: Description of what the user wants to do
+            agent_id: Optional agent ID for filtering (reserved for future use)
+            search_mode: How to search for tools:
+                - "llm": Use LLM to recommend (default, original behavior)
+                - "semantic": Use embedding similarity search
+                - "hybrid": Combine semantic search with LLM ranking
+            top_k: Maximum recommendations to return (for semantic/hybrid)
+            min_similarity: Minimum similarity threshold (for semantic/hybrid)
+            project_id: Project ID for semantic/hybrid search (overrides instance default)
+
+        Returns:
+            Dict with recommendations and metadata
+        """
+        # Use provided project_id or fall back to instance default
+        effective_project_id = project_id or self._project_id
+
+        if search_mode == "semantic":
+            return await self._recommend_semantic(
+                task_description, top_k, min_similarity, effective_project_id
+            )
+        elif search_mode == "hybrid":
+            return await self._recommend_hybrid(
+                task_description, top_k, min_similarity, effective_project_id
+            )
+        else:
+            return await self._recommend_llm(task_description)
+
+    async def _recommend_semantic(
+        self, task_description: str, top_k: int, min_similarity: float, project_id: str | None
+    ) -> dict[str, Any]:
+        """Recommend tools using semantic similarity search."""
+        if not self._semantic_search:
+            return {
+                "success": False,
+                "error": "Semantic search not configured",
+                "task": task_description,
+            }
+
+        if not project_id:
+            return {
+                "success": False,
+                "error": "Project ID not set for semantic search",
+                "task": task_description,
+            }
+
+        try:
+            results = await self._semantic_search.search_tools(
+                query=task_description,
+                project_id=project_id,
+                top_k=top_k,
+                min_similarity=min_similarity,
+            )
+
+            recommendations = [
+                {
+                    "server": r.server_name,
+                    "tool": r.tool_name,
+                    "reason": r.description or "Semantically similar to query",
+                    "similarity": round(r.similarity, 4),
+                }
+                for r in results
+            ]
+
+            return {
+                "success": True,
+                "task": task_description,
+                "search_mode": "semantic",
+                "recommendation": recommendations,
+                "recommendations": recommendations,
+                "total_results": len(results),
+            }
+        except Exception as e:
+            logger.error(f"Semantic search failed: {e}")
+            return {"success": False, "error": str(e), "task": task_description}
+
+    async def _recommend_hybrid(
+        self, task_description: str, top_k: int, min_similarity: float, project_id: str | None
+    ) -> dict[str, Any]:
+        """Recommend tools using semantic search + LLM re-ranking."""
+        # First get semantic results
+        semantic_result = await self._recommend_semantic(
+            task_description,
+            top_k * 2,
+            min_similarity,
+            project_id,  # Get more for re-ranking
+        )
+
+        if not semantic_result.get("success") or not semantic_result.get("recommendations"):
+            # Fall back to pure LLM if semantic fails
+            return await self._recommend_llm(task_description)
+
+        # Use LLM to re-rank and add reasoning
+        try:
+            config = self._get_config()
+            candidates = semantic_result["recommendations"]
+            candidate_list = "\n".join(
+                f"- {c['server']}/{c['tool']}: {c.get('reason', 'No description')}"
+                for c in candidates
+            )
+
+            prompt = config.hybrid_rerank_prompt.format(
+                task_description=task_description,
+                candidate_list=candidate_list,
+                top_k=top_k,
+            )
+
+            provider = self._llm_service.get_default_provider()
+            response = await provider.generate_text(prompt)
+
+            # Parse LLM response
+            if "```json" in response:
+                response = response.split("```json")[1].split("```")[0].strip()
+            elif "```" in response:
+                response = response.split("```")[1].split("```")[0].strip()
+
+            data = json.loads(response)
+            recommendations = data.get("recommendations", [])[:top_k]
+
+            return {
+                "success": True,
+                "task": task_description,
+                "search_mode": "hybrid",
+                "recommendation": recommendations,
+                "recommendations": recommendations,
+                "semantic_candidates": len(candidates),
+            }
+        except Exception as e:
+            logger.warning(f"Hybrid LLM re-ranking failed, using semantic results: {e}")
+            # Fall back to semantic results
+            semantic_result["search_mode"] = "hybrid_fallback"
+            return semantic_result
+
+    async def _recommend_llm(self, task_description: str) -> dict[str, Any]:
+        """Recommend tools using LLM (original behavior)."""
+        try:
+            config = self._get_config()
+            available_servers = self._mcp_manager.get_available_servers()
+
+            prompt = config.llm_prompt.format(
+                task_description=task_description,
+                available_servers=", ".join(available_servers),
+            )
+
+            provider = self._llm_service.get_default_provider()
+            response = await provider.generate_text(prompt)
+
+            try:
+                if "```json" in response:
+                    response = response.split("```json")[1].split("```")[0].strip()
+                elif "```" in response:
+                    response = response.split("```")[1].split("```")[0].strip()
+
+                data = json.loads(response)
+                recommendations = data.get("recommendations", [])
+            except (json.JSONDecodeError, KeyError, IndexError) as e:
+                recommendations = []
+                logger.warning(f"Failed to parse LLM recommendation response: {e}")
+
+            return {
+                "success": True,
+                "task": task_description,
+                "search_mode": "llm",
+                "recommendation": recommendations,
+                "recommendations": recommendations,
+                "available_servers": available_servers,
+            }
+        except Exception as e:
+            logger.error(f"Error generating recommendations: {e}")
+            return {"success": False, "error": str(e), "task": task_description}