universal-mcp 0.1.24rc14__tar.gz → 0.1.24rc19__tar.gz

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: universal-mcp
-Version: 0.1.24rc14
+Version: 0.1.24rc19
 Summary: Universal MCP acts as a middle ware for your API applications. It can store your credentials, authorize, enable disable apps on the fly and much more.
 Author-email: Manoj Bajaj <manojbajaj95@gmail.com>
 License: MIT

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "universal-mcp"
-version = "0.1.24-rc14"
+version = "0.1.24-rc19"
 description = "Universal MCP acts as a middle ware for your API applications. It can store your credentials, authorize, enable disable apps on the fly and much more."
 readme = "README.md"
 authors = [

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/tests/test_api_integration.py

@@ -1,7 +1,6 @@
 import pytest
 
-from universal_mcp.applications import app_from_config
-from universal_mcp.config import AppConfig  # <-- Import the AppConfig model
+from universal_mcp.applications.utils import app_from_slug
 from universal_mcp.exceptions import NotAuthorizedError
 from universal_mcp.integrations import ApiKeyIntegration
 from universal_mcp.stores import MemoryStore
@@ -11,8 +10,7 @@ def test_perplexity_api_no_key():
     store = MemoryStore()
 
     integration = ApiKeyIntegration("PERPLEXITY", store=store)
-    perplexity_app_config = AppConfig(name="perplexity")
-    PerplexityApp = app_from_config(perplexity_app_config)
+    PerplexityApp = app_from_slug("perplexity")
 
     app = PerplexityApp(integration=integration)
 

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/universal_mcp/agentr/client.py

@@ -1,3 +1,4 @@
+import io
 import os
 from typing import Any
 
@@ -25,6 +26,7 @@ class AgentrClient:
         base_url = base_url or os.getenv("AGENTR_BASE_URL", "https://api.agentr.dev")
         self.base_url = f"{base_url.rstrip('/')}/v1"
         api_key = api_key or os.getenv("AGENTR_API_KEY")
+        self.user_id = None
         if api_key:
             self.client = httpx.Client(
                 base_url=self.base_url,
@@ -34,6 +36,7 @@ class AgentrClient:
                 verify=False,
             )
             me_data = self.me()
+            self.user_id = me_data["id"]
             logger.debug(f"Client initialized with user: {me_data['email']}")
         elif auth_token:
             logger.debug("Initializing client with auth token")
@@ -45,6 +48,7 @@ class AgentrClient:
                 verify=False,
             )
             me_data = self.me()
+            self.user_id = me_data["id"]
             logger.debug(f"Client initialized with user: {me_data['email']}")
         else:
             raise ValueError("No API key or auth token provided")
@@ -207,3 +211,10 @@ class AgentrClient:
         response = self.client.get("/tools/", params=params)
         response.raise_for_status()
         return response.json().get("items", [])
+
+    def _upload_file(self, file_name: str, mime_type: str, base64_data: str) -> str:
+        """Upload a file to the server."""
+        files = {"file": (file_name, io.BytesIO(base64_data), mime_type)}
+        reponse = self.client.post("/files/upload", files=files)
+        reponse.raise_for_status()
+        return reponse.json()

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/universal_mcp/agentr/registry.py

@@ -1,15 +1,34 @@
+import base64
 from typing import Any
 
 from loguru import logger
 
 from universal_mcp.agentr.client import AgentrClient
-from universal_mcp.applications import app_from_slug
+from universal_mcp.applications.utils import app_from_slug
+from universal_mcp.exceptions import ToolError
 from universal_mcp.tools.manager import ToolManager, _get_app_and_tool_name
 from universal_mcp.tools.registry import ToolRegistry
 from universal_mcp.types import ToolConfig, ToolFormat
 
 from .integration import AgentrIntegration
 
+MARKDOWN_INSTRUCTIONS = """Always render the URL in markdown format for images and media files. Here are examples:
+The url is provided in the response as "signed_url".
+For images:
+- Use markdown image syntax: ![alt text](url)
+- Example: ![Generated sunset image](https://example.com/image.png)
+- Always include descriptive alt text that explains what the image shows
+
+For audio files:
+- Use markdown link syntax with audio description: [🔊 Audio file description](url)
+- Example: [🔊 Generated speech audio](https://example.com/audio.mp3)
+
+For other media:
+- Use descriptive link text: [📄 File description](url)
+- Example: [📄 Generated document](https://example.com/document.pdf)
+
+Always make the links clickable and include relevant context about what the user will see or hear when they access the URL."""
+
 
 class AgentrRegistry(ToolRegistry):
     """Platform manager implementation for AgentR platform."""
@@ -133,7 +152,7 @@ class AgentrRegistry(ToolRegistry):
         try:
             # Clear tools from tool manager before loading new tools
             self.tool_manager.clear_tools()
-            if isinstance(tools, ToolConfig):
+            if isinstance(tools, dict):
                 logger.info("Loading tools from tool config")
                 self._load_tools_from_tool_config(tools, self.tool_manager)
             else:
@@ -176,12 +195,28 @@ class AgentrRegistry(ToolRegistry):
             tool_config: The tool configuration containing app names and tools
             tool_manager: The tool manager to register tools with
         """
-        for app_name, tool_data in tool_config.agentrServers.items():
-            self._load_tools(app_name, tool_data.tools, tool_manager)
+        for app_name, tool_names in tool_config.items():
+            self._load_tools(app_name, tool_names, tool_manager)
 
     async def call_tool(self, tool_name: str, tool_args: dict[str, Any]) -> dict[str, Any]:
         """Call a tool with the given name and arguments."""
-        return await self.tool_manager.call_tool(tool_name, tool_args)
+        data = await self.tool_manager.call_tool(tool_name, tool_args)
+        logger.debug(f"Tool {tool_name} called with args {tool_args} and returned {data}")
+        if isinstance(data, dict):
+            type_ = data.get("type")
+            if type_ == "image" or type_ == "audio":
+                # Special handling for images and audio
+                base64_data = data.get("data")
+                mime_type = data.get("mime_type")
+                file_name = data.get("file_name")
+                if not mime_type or not file_name:
+                    raise ToolError("Mime type or file name is missing")
+                bytes_data = base64.b64decode(base64_data)
+                response = self.client._upload_file(file_name, mime_type, bytes_data)
+                # Hard code instructions for llm
+                response = {**response, "instructions": MARKDOWN_INSTRUCTIONS}
+                return response
+        return data
 
     async def list_connected_apps(self) -> list[str]:
         """List all apps that the user has connected."""

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/universal_mcp/applications/application.py

@@ -9,7 +9,6 @@ from gql.transport.requests import RequestsHTTPTransport
 from graphql import DocumentNode
 from loguru import logger
 
-from universal_mcp.analytics import analytics
 from universal_mcp.integrations.integration import Integration
 
 
@@ -37,7 +36,6 @@ class BaseApplication(ABC):
         """
         self.name = name
         logger.debug(f"Initializing Application '{name}' with kwargs: {kwargs}")
-        analytics.track_app_loaded(name)  # Track app loading
 
     @abstractmethod
     def list_tools(self) -> list[Callable]:

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/universal_mcp/applications/sample/app.py

@@ -1,22 +1,25 @@
 import datetime
 
 import httpx
+from loguru import logger
 
 from universal_mcp.applications.application import BaseApplication
 
 
-class SampleToolApp(BaseApplication):
+class SampleApp(BaseApplication):
     """A sample application providing basic utility tools."""
 
-    def __init__(self):
+    def __init__(self, **kwargs):
         """Initializes the SampleToolApp with the name 'sample_tool_app'."""
-        super().__init__(name="sample_tool_app")
+        super().__init__(name="sample")
 
     def get_current_time(self):
         """Get the current system time as a formatted string.
 
         Returns:
             str: The current time in the format 'YYYY-MM-DD HH:MM:SS'.
+        Tags:
+            time, date, current, system, utility, important
         """
         return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
@@ -25,6 +28,8 @@ class SampleToolApp(BaseApplication):
 
         Returns:
             str: The current date in the format 'YYYY-MM-DD'.
+        Tags:
+            time, date, current, system, utility, important
         """
         return datetime.datetime.now().strftime("%Y-%m-%d")
 
@@ -36,6 +41,8 @@ class SampleToolApp(BaseApplication):
 
         Returns:
             str: The result of the calculation, or an error message if evaluation fails.
+        Tags:
+            math, calculation, utility, important
         """
         try:
             # Safe evaluation of mathematical expressions
@@ -44,29 +51,41 @@ class SampleToolApp(BaseApplication):
         except Exception as e:
             return f"Error in calculation: {str(e)}"
 
-    def file_operations(self, operation: str, filename: str, content: str = ""):
-        """Perform file read or write operations.
+    def read_file(self, filename: str):
+        """Read content from a file.
 
         Args:
-            operation (str): The operation to perform, either 'read' or 'write'.
-            filename (str): The name of the file to operate on.
-            content (str, optional): The content to write to the file (used only for 'write'). Defaults to "".
+            filename (str): The name of the file to read from.
 
         Returns:
-            str: The result of the file operation, or an error message if the operation fails.
+            str: The content of the file, or an error message if the operation fails.
+        Tags:
+            file, read, utility, important
         """
         try:
-            if operation == "read":
-                with open(filename) as f:
-                    return f"File content:\n{f.read()}"
-            elif operation == "write":
-                with open(filename, "w") as f:
-                    f.write(content)
-                return f"Successfully wrote to {filename}"
-            else:
-                return "Invalid operation. Use 'read' or 'write'"
+            with open(filename) as f:
+                return f"File content:\n{f.read()}"
         except Exception as e:
-            return f"File operation error: {str(e)}"
+            return f"File read error: {str(e)}"
+
+    def write_file(self, filename: str, content: str):
+        """Write content to a file.
+
+        Args:
+            filename (str): The name of the file to write to.
+            content (str): The content to write to the file.
+
+        Returns:
+            str: Success message or an error message if the operation fails.
+        Tags:
+            file, write, utility, important
+        """
+        try:
+            with open(filename, "w") as f:
+                f.write(content)
+            return f"Successfully wrote to {filename}"
+        except Exception as e:
+            return f"File write error: {str(e)}"
 
     def get_weather(
         self,
@@ -230,6 +249,44 @@ class SampleToolApp(BaseApplication):
         except Exception as e:
             return {"error": str(e)}
 
+    def generate_image(self, prompt: str):
+        """Generate an image based on a prompt.
+        A
+        Args:
+            prompt (str): The prompt to generate an image from.
+
+        Returns:
+            dict: The generated image.
+        Tags:
+            image, generate, utility, important
+        """
+        import base64
+        import io
+
+        from PIL import Image, ImageDraw, ImageFont
+
+        # Create a simple placeholder image
+        img = Image.new("RGB", (600, 400), color="lightblue")
+        draw = ImageDraw.Draw(img)
+
+        # Add text to the image
+        try:
+            # Try to use a default font
+            font = ImageFont.load_default()
+        except Exception as e:
+            logger.error(f"Error loading font: {e}")
+            font = None
+
+        text = f"Generated: {prompt[:50]}..."
+        draw.text((50, 200), text, fill="black", font=font)
+
+        # Convert to base64
+        buffer = io.BytesIO()
+        img.save(buffer, format="PNG")
+        img_base64 = base64.b64encode(buffer.getvalue()).decode("utf-8")
+
+        return {"type": "image", "data": img_base64, "mime_type": "image/png", "file_name": "sample.png"}
+
     def list_tools(self):
         """List all available tool methods in this application.
 
@@ -240,6 +297,8 @@ class SampleToolApp(BaseApplication):
             self.get_current_time,
             self.get_current_date,
             self.calculate,
-            self.file_operations,
+            self.read_file,
+            self.write_file,
             self.get_simple_weather,
+            self.generate_image,
         ]

universal_mcp-0.1.24rc19/src/universal_mcp/applications/utils.py

@@ -0,0 +1,52 @@
+import importlib
+
+from loguru import logger
+
+from universal_mcp.applications.application import BaseApplication
+
+# --- Default Name Generators ---
+
+
+def get_default_package_name(slug: str) -> str:
+    """
+    Convert a repository slug to a package name.
+    """
+    slug = slug.strip().lower().replace("-", "_")
+    package_name = f"universal_mcp.applications.{slug}"
+    return package_name
+
+
+def get_default_module_path(slug: str) -> str:
+    """
+    Convert a repository slug to a module path.
+    """
+    package_name = get_default_package_name(slug)
+    module_path = f"{package_name}.app"
+    return module_path
+
+
+def get_default_class_name(slug: str) -> str:
+    """
+    Convert a repository slug to a class name.
+    """
+    slug = slug.strip().lower()
+    parts = slug.replace("-", "_").split("_")
+    class_name = "".join(part.capitalize() for part in parts) + "App"
+    return class_name
+
+
+# --- Application Loaders ---
+
+
+def app_from_slug(slug: str) -> type[BaseApplication]:
+    """Loads an application from a slug."""
+    return load_app_from_package(slug)
+
+
+def load_app_from_package(slug: str) -> type[BaseApplication]:
+    """Loads an application from a pip-installable package."""
+    logger.debug(f"Loading '{slug}' as a package.")
+    module_path_str = get_default_module_path(slug)
+    class_name_str = get_default_class_name(slug)
+    module = importlib.import_module(module_path_str)
+    return getattr(module, class_name_str)

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/universal_mcp/servers/server.py

@@ -5,7 +5,8 @@ from loguru import logger
 from mcp.server.fastmcp import FastMCP
 from mcp.types import TextContent
 
-from universal_mcp.applications import BaseApplication, app_from_config
+from universal_mcp.applications.application import BaseApplication
+from universal_mcp.applications.utils import app_from_slug
 from universal_mcp.config import ServerConfig
 from universal_mcp.exceptions import ConfigurationError, ToolError
 from universal_mcp.integrations.integration import ApiKeyIntegration, OAuthIntegration
@@ -44,8 +45,8 @@ def load_from_local_config(config: ServerConfig, tool_manager: ToolManager) -> N
                     integration = OAuthIntegration(config.name, store=store, **app_config.integration.credentials)
                 else:
                     raise ValueError(f"Unsupported integration type: {app_config.integration.type}")
-            app = app_from_config(app_config)(integration=integration)
-            tool_manager.register_tools_from_app(app, app_config.actions)
+            app = app_from_slug(app_config.name)(integration=integration)
+            tool_manager.register_tools_from_app(app, tool_names=app_config.actions)
             logger.info(f"Loaded app: {app_config.name}")
         except Exception as e:
             logger.error(f"Failed to load app {app_config.name}: {e}", exc_info=True)

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/universal_mcp/tools/manager.py

@@ -3,7 +3,6 @@ from typing import Any
 
 from loguru import logger
 
-from universal_mcp.analytics import analytics
 from universal_mcp.applications.application import BaseApplication
 from universal_mcp.exceptions import ToolNotFoundError
 from universal_mcp.tools.adapters import (
@@ -298,8 +297,6 @@ class ToolManager:
             raise ToolNotFoundError(f"Unknown tool: {name}")
         try:
             result = await tool.run(arguments, context)
-            analytics.track_tool_called(name, app_name, "success")
             return result
         except Exception as e:
-            analytics.track_tool_called(name, app_name, "error", str(e))
             raise e

universal_mcp-0.1.24rc19/src/universal_mcp/types.py

@@ -0,0 +1,18 @@
+from enum import Enum
+
+# Constants
+DEFAULT_IMPORTANT_TAG = "important"
+TOOL_NAME_SEPARATOR = "__"
+DEFAULT_APP_NAME = "common"
+
+
+class ToolFormat(str, Enum):
+    """Supported tool formats."""
+
+    NATIVE = "native"
+    MCP = "mcp"
+    LANGCHAIN = "langchain"
+    OPENAI = "openai"
+
+
+ToolConfig = dict[str, list[str]]

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/universal_mcp/utils/prompts.py

@@ -229,7 +229,6 @@ from loguru import logger
 
 import httpx
 
-from universal_mcp.analytics import analytics
 from universal_mcp.integrations import Integration
 
 class BaseApplication(ABC):
@@ -256,7 +255,6 @@ class BaseApplication(ABC):
         """
         self.name = name
         logger.debug(f"Initializing Application '{name}' with kwargs: {kwargs}")
-        analytics.track_app_loaded(name)  # Track app loading
 
     @abstractmethod
     def list_tools(self) -> list[Callable]:

{universal_mcp-0.1.24rc14 → universal_mcp-0.1.24rc19}/src/universal_mcp/utils/testing.py

@@ -8,7 +8,7 @@ from pydantic import BaseModel, SecretStr
 
 from universal_mcp.agentr import AgentrIntegration
 from universal_mcp.agentr.client import AgentrClient
-from universal_mcp.applications import APIApplication, BaseApplication
+from universal_mcp.applications.application import APIApplication, BaseApplication
 from universal_mcp.tools import Tool, ToolManager
 from universal_mcp.types import ToolFormat
 

universal_mcp-0.1.24rc14/src/evals/dataset.py

@@ -1,24 +0,0 @@
-import csv
-import json
-from typing import Any
-
-
-def load_dataset(file_path: str) -> list[dict[str, Any]]:
-    """
-    Loads a dataset from a CSV or JSONL file.
-
-    Args:
-        file_path: The path to the dataset file.
-
-    Returns:
-        A list of dictionaries, where each dictionary represents an example.
-    """
-    if file_path.endswith(".csv"):
-        with open(file_path, encoding="utf-8") as f:
-            reader = csv.DictReader(f)
-            return list(reader)
-    elif file_path.endswith(".jsonl"):
-        with open(file_path, encoding="utf-8") as f:
-            return [json.loads(line) for line in f]
-    else:
-        raise ValueError("Unsupported file format. Please use CSV or JSONL.")

universal_mcp-0.1.24rc14/src/evals/datasets/creative_writing.jsonl

@@ -1,3 +0,0 @@
-{"user_input": "Write a haiku about a rainy day."}
-{"user_input": "Tell me a short story about a friendly ghost."}
-{"user_input": "Write a poem about the changing seasons."}

universal_mcp-0.1.24rc14/src/evals/datasets/exact.jsonl

@@ -1,6 +0,0 @@
-{"user_input": "What is 2 + 2?", "expected_output": "4"}
-{"user_input": "What is the opposite of 'hot'?", "expected_output": "cold"}
-{"user_input": "Who is the first president of the United States?", "expected_output": "George Washington"}
-{"user_input": "What is the capital of France?", "expected_output": "Paris"}
-{"user_input": "Who wrote 'To Kill a Mockingbird'?", "expected_output": "Harper Lee"}
-{"user_input": "What is the boiling point of water at sea level in Celsius?", "expected_output": "100"}

universal_mcp-0.1.24rc14/src/evals/datasets/tasks.jsonl

@@ -1,22 +0,0 @@
-{"user_input": "Send an email to manoj@agentr.dev from my Gmail account", "difficulty": 1, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["send_email"]}}}
-{"user_input": "Show me events from today's Google Calendar.", "difficulty": 1, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["list_events"]}}}}
-{"user_input": "Create a Google Doc summarizing the last 5 merged pull requests in my GitHub repo- universal-mcp/universal-mcp, including links and commit highlights.", "difficulty": 4, "expected_tools": {"agentrServers": {"github": {"tools": ["get_pull_request"]}, "google-docs": {"tools": ["create_document"]}}}}
-{"user_input": "Summarize the key insights from all marketing emails received this week from my Gmail and add a section in a Google Doc with action points.", "difficulty": 4, "expected_tools": {"agentrServers": {"google-gemini": {"tools": ["prompt_document"]}, "google-docs": {"tools": ["style_text"]}}}}
-{"user_input": "Create a Google Sheet of the best cafes and restaurants near IIT Bombay", "difficulty": 3, "expected_tools": {"agentrServers": {"google-sheet": {"tools": ["create_spreadsheet"]}, "serpapi": {"tools": ["google_maps_search"]}}}}
-{"user_input": "Track the top posts in r/startups over the past 7 days using Reddit and create a trend report on what's being discussed most (e.g., hiring, funding, MVPs) in a Google Doc.", "difficulty": 5, "expected_tools": {"agentrServers": {"reddit": {"tools": ["search"]}, "google-docs": {"tools": ["add_content", "create_footer"]}}}}
-{"user_input": "Find the best restaurants in Goa using perplexity web search", "difficulty": 2, "expected_tools": {"agentrServers": {}}}
-{"user_input": "List the unread emails from the last 24 hours from my Gmail, sorted by sender.", "difficulty": 2, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["list_messages"]}}}}
-{"user_input": "Tell me how many meetings I have tomorrow and when they start from my Google Calendar.", "difficulty": 1, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["get_today_events"]}}}}
-{"user_input": "Create a meeting with aditakarsh@example.com on the topic of the latest trends in AI at 8PM today using Google Calendar.", "difficulty": 2, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["add_an_event"]}}}}
-{"user_input": "What are the topics of my meetings today from Google Calendar and who are the attendees? Give a 1-line context for each attendee using LinkedIn or web search.", "difficulty": 4, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["list_events"]}, "serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}
-{"user_input": "Fetch my last inbox mail from Microsoft Outlook", "difficulty": 1, "expected_tools": {"agentrServers": {"outlook": {"tools": ["user_get_mail_folder"]}}}}
-{"user_input": "Fetch unsubscribe links from my Gmail inbox for promo emails I have received in the last 7 days", "difficulty": 3, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["delete_filters", "list_filters"]}}}}
-{"user_input": "Fetch all unread emails from Gmail and new tickets from ClickUp for me from last night", "difficulty": 4, "expected_tools": {"agentrServers": {"clickup": {"tools": ["tasks_filter_team_tasks"]}}}}
-{"user_input": "Give me a report on the earnings of Oklo using web search, and projections for the company revenue, stock price", "difficulty": 4, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}
-{"user_input": "Create a weekly expense report from my credit card transactions and categorize spending by type (food, transport, entertainment, etc.) in a Google Sheet", "difficulty": 3, "expected_tools": {"agentrServers": {"google-sheet": {"tools": ["create_spreadsheet"]}}}}
-{"user_input": "Generate a comparison table of SaaS tools for project management using web search, including pricing, features, and user ratings in a Google Sheet", "difficulty": 4, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "google-sheet": {"tools": ["add_table", "list_tables"]}}}}
-{"user_input": "Research the top 10 Y Combinator startups from the latest batch using web search and create a report on their industries and funding status in Google Docs", "difficulty": 5, "expected_tools": {"agentrServers": {"google-docs": {"tools": ["create_footer", "create_header"]}}}}
-{"user_input": "Find and summarize the key takeaways from the latest earnings calls of FAANG companies using web search and create a report in Google Docs", "difficulty": 5, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "google-docs": {"tools": ["style_text", "create_footer"]}}}}
-{"user_input": "Draft personalized LinkedIn outreach messages for 10 potential collaborators in the fintech space based on their recent posts using LinkedIn data in a Google Sheet", "difficulty": 5, "expected_tools": {"agentrServers": {}}}
-{"user_input": "Monitor my Twitter mentions and DMs from the past 48 hours and create a response priority list in Google Sheets", "difficulty": 4, "expected_tools": {"agentrServers": {"twitter": {"tools": ["search_stream"]}, "google-sheet": {"tools": ["add_sheet", "add_table"]}}}}
-{"user_input": "Create a content calendar for next month with trending AI/ML topics using web search and optimal posting times based on my audience analytics in Google Sheets", "difficulty": 5, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["add_an_event"]}, "google-sheet": {"tools": ["append_values"]}, "serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}

universal_mcp-0.1.24rc14/src/evals/evaluators.py

@@ -1,40 +0,0 @@
-from langsmith.evaluation import EvaluationResult, run_evaluator
-from langsmith.schemas import Example, Run
-from openevals.llm import create_llm_as_judge
-from openevals.prompts import CORRECTNESS_PROMPT
-
-
-@run_evaluator
-def exact_match_evaluator(run: Run, example: Example | None = None) -> EvaluationResult:
-    """
-    A simple evaluator that checks for exact match between the agent's output
-    and the expected output from the dataset.
-    """
-    if example is None or "expected_output" not in example.outputs:
-        return EvaluationResult(key="exact_match", score=0, comment="No expected output provided.")
-
-    # The agent's response might be in a list of messages
-    agent_response_raw = run.outputs.get("output", "")
-    if isinstance(agent_response_raw, list):
-        # Extract text from the last dictionary in the list
-        agent_response = agent_response_raw[-1].get("text", "").strip() if agent_response_raw else ""
-    else:
-        agent_response = str(agent_response_raw).strip()
-
-    expected_output = example.outputs["expected_output"].strip()
-
-    if agent_response == expected_output:
-        score = 1
-        comment = "Exact match."
-    else:
-        score = 0
-        comment = f"Mismatch: Expected '{expected_output}', but got '{agent_response}'."
-
-    return EvaluationResult(key="exact_match", score=score, comment=comment)
-
-
-correctness_evaluator = create_llm_as_judge(
-    prompt=CORRECTNESS_PROMPT,
-    feedback_key="correctness",
-    model="anthropic:claude-4-sonnet-20250514",
-)