universal-mcp 0.1.24rc13__tar.gz → 0.1.24rc17__tar.gz

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: universal-mcp
-Version: 0.1.24rc13
+Version: 0.1.24rc17
 Summary: Universal MCP acts as a middle ware for your API applications. It can store your credentials, authorize, enable disable apps on the fly and much more.
 Author-email: Manoj Bajaj <manojbajaj95@gmail.com>
 License: MIT
@@ -32,6 +32,7 @@ Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: rich>=14.0.0
 Requires-Dist: streamlit>=1.46.1
 Requires-Dist: typer>=0.15.2
+Requires-Dist: universal-mcp-applications>=0.1.2
 Provides-Extra: dev
 Requires-Dist: litellm>=1.30.7; extra == 'dev'
 Requires-Dist: pre-commit>=4.2.0; extra == 'dev'

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "universal-mcp"
-version = "0.1.24-rc13"
+version = "0.1.24-rc17"
 description = "Universal MCP acts as a middle ware for your API applications. It can store your credentials, authorize, enable disable apps on the fly and much more."
 readme = "README.md"
 authors = [
@@ -39,6 +39,7 @@ dependencies = [
     "rich>=14.0.0",
     "streamlit>=1.46.1",
     "typer>=0.15.2",
+    "universal-mcp-applications>=0.1.2",
 ]
 
 [project.optional-dependencies]

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/src/tests/test_api_integration.py

@@ -1,7 +1,6 @@
 import pytest
 
-from universal_mcp.applications import app_from_config
-from universal_mcp.config import AppConfig  # <-- Import the AppConfig model
+from universal_mcp.applications.utils import app_from_slug
 from universal_mcp.exceptions import NotAuthorizedError
 from universal_mcp.integrations import ApiKeyIntegration
 from universal_mcp.stores import MemoryStore
@@ -11,8 +10,7 @@ def test_perplexity_api_no_key():
     store = MemoryStore()
 
     integration = ApiKeyIntegration("PERPLEXITY", store=store)
-    perplexity_app_config = AppConfig(name="perplexity")
-    PerplexityApp = app_from_config(perplexity_app_config)
+    PerplexityApp = app_from_slug("perplexity")
 
     app = PerplexityApp(integration=integration)
 

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/src/universal_mcp/agentr/registry.py

@@ -3,7 +3,7 @@ from typing import Any
 from loguru import logger
 
 from universal_mcp.agentr.client import AgentrClient
-from universal_mcp.applications import app_from_slug
+from universal_mcp.applications.utils import app_from_slug
 from universal_mcp.tools.manager import ToolManager, _get_app_and_tool_name
 from universal_mcp.tools.registry import ToolRegistry
 from universal_mcp.types import ToolConfig, ToolFormat
@@ -133,7 +133,7 @@ class AgentrRegistry(ToolRegistry):
         try:
             # Clear tools from tool manager before loading new tools
             self.tool_manager.clear_tools()
-            if isinstance(tools, ToolConfig):
+            if isinstance(tools, dict):
                 logger.info("Loading tools from tool config")
                 self._load_tools_from_tool_config(tools, self.tool_manager)
             else:
@@ -176,8 +176,8 @@ class AgentrRegistry(ToolRegistry):
             tool_config: The tool configuration containing app names and tools
             tool_manager: The tool manager to register tools with
         """
-        for app_name, tool_data in tool_config.agentrServers.items():
-            self._load_tools(app_name, tool_data.tools, tool_manager)
+        for app_name, tool_names in tool_config.items():
+            self._load_tools(app_name, tool_names, tool_manager)
 
     async def call_tool(self, tool_name: str, tool_args: dict[str, Any]) -> dict[str, Any]:
         """Call a tool with the given name and arguments."""

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/src/universal_mcp/applications/application.py

@@ -9,7 +9,6 @@ from gql.transport.requests import RequestsHTTPTransport
 from graphql import DocumentNode
 from loguru import logger
 
-from universal_mcp.analytics import analytics
 from universal_mcp.integrations.integration import Integration
 
 
@@ -37,7 +36,6 @@ class BaseApplication(ABC):
         """
         self.name = name
         logger.debug(f"Initializing Application '{name}' with kwargs: {kwargs}")
-        analytics.track_app_loaded(name)  # Track app loading
 
     @abstractmethod
     def list_tools(self) -> list[Callable]:

universal_mcp-0.1.24rc17/src/universal_mcp/applications/utils.py

@@ -0,0 +1,52 @@
+import importlib
+
+from loguru import logger
+
+from universal_mcp.applications.application import BaseApplication
+
+# --- Default Name Generators ---
+
+
+def get_default_package_name(slug: str) -> str:
+    """
+    Convert a repository slug to a package name.
+    """
+    slug = slug.strip().lower().replace("-", "_")
+    package_name = f"universal_mcp.applications.{slug}"
+    return package_name
+
+
+def get_default_module_path(slug: str) -> str:
+    """
+    Convert a repository slug to a module path.
+    """
+    package_name = get_default_package_name(slug)
+    module_path = f"{package_name}.app"
+    return module_path
+
+
+def get_default_class_name(slug: str) -> str:
+    """
+    Convert a repository slug to a class name.
+    """
+    slug = slug.strip().lower()
+    parts = slug.replace("-", "_").split("_")
+    class_name = "".join(part.capitalize() for part in parts) + "App"
+    return class_name
+
+
+# --- Application Loaders ---
+
+
+def app_from_slug(slug: str) -> type[BaseApplication]:
+    """Loads an application from a slug."""
+    return load_app_from_package(slug)
+
+
+def load_app_from_package(slug: str) -> type[BaseApplication]:
+    """Loads an application from a pip-installable package."""
+    logger.debug(f"Loading '{slug}' as a package.")
+    module_path_str = get_default_module_path(slug)
+    class_name_str = get_default_class_name(slug)
+    module = importlib.import_module(module_path_str)
+    return getattr(module, class_name_str)

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/src/universal_mcp/servers/server.py

@@ -5,7 +5,8 @@ from loguru import logger
 from mcp.server.fastmcp import FastMCP
 from mcp.types import TextContent
 
-from universal_mcp.applications import BaseApplication, app_from_config
+from universal_mcp.applications.application import BaseApplication
+from universal_mcp.applications.utils import app_from_slug
 from universal_mcp.config import ServerConfig
 from universal_mcp.exceptions import ConfigurationError, ToolError
 from universal_mcp.integrations.integration import ApiKeyIntegration, OAuthIntegration
@@ -44,8 +45,8 @@ def load_from_local_config(config: ServerConfig, tool_manager: ToolManager) -> N
                     integration = OAuthIntegration(config.name, store=store, **app_config.integration.credentials)
                 else:
                     raise ValueError(f"Unsupported integration type: {app_config.integration.type}")
-            app = app_from_config(app_config)(integration=integration)
-            tool_manager.register_tools_from_app(app, app_config.actions)
+            app = app_from_slug(app_config.name)(integration=integration)
+            tool_manager.register_tools_from_app(app, tool_names=app_config.actions)
             logger.info(f"Loaded app: {app_config.name}")
         except Exception as e:
             logger.error(f"Failed to load app {app_config.name}: {e}", exc_info=True)

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/src/universal_mcp/tools/manager.py

@@ -3,7 +3,6 @@ from typing import Any
 
 from loguru import logger
 
-from universal_mcp.analytics import analytics
 from universal_mcp.applications.application import BaseApplication
 from universal_mcp.exceptions import ToolNotFoundError
 from universal_mcp.tools.adapters import (
@@ -298,8 +297,6 @@ class ToolManager:
             raise ToolNotFoundError(f"Unknown tool: {name}")
         try:
             result = await tool.run(arguments, context)
-            analytics.track_tool_called(name, app_name, "success")
             return result
         except Exception as e:
-            analytics.track_tool_called(name, app_name, "error", str(e))
             raise e

universal_mcp-0.1.24rc17/src/universal_mcp/types.py

@@ -0,0 +1,18 @@
+from enum import Enum
+
+# Constants
+DEFAULT_IMPORTANT_TAG = "important"
+TOOL_NAME_SEPARATOR = "__"
+DEFAULT_APP_NAME = "common"
+
+
+class ToolFormat(str, Enum):
+    """Supported tool formats."""
+
+    NATIVE = "native"
+    MCP = "mcp"
+    LANGCHAIN = "langchain"
+    OPENAI = "openai"
+
+
+ToolConfig = dict[str, list[str]]

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/src/universal_mcp/utils/prompts.py

@@ -229,7 +229,6 @@ from loguru import logger
 
 import httpx
 
-from universal_mcp.analytics import analytics
 from universal_mcp.integrations import Integration
 
 class BaseApplication(ABC):
@@ -256,7 +255,6 @@ class BaseApplication(ABC):
         """
         self.name = name
         logger.debug(f"Initializing Application '{name}' with kwargs: {kwargs}")
-        analytics.track_app_loaded(name)  # Track app loading
 
     @abstractmethod
     def list_tools(self) -> list[Callable]:

{universal_mcp-0.1.24rc13 → universal_mcp-0.1.24rc17}/src/universal_mcp/utils/testing.py

@@ -8,7 +8,7 @@ from pydantic import BaseModel, SecretStr
 
 from universal_mcp.agentr import AgentrIntegration
 from universal_mcp.agentr.client import AgentrClient
-from universal_mcp.applications import APIApplication, BaseApplication
+from universal_mcp.applications.application import APIApplication, BaseApplication
 from universal_mcp.tools import Tool, ToolManager
 from universal_mcp.types import ToolFormat
 

universal_mcp-0.1.24rc13/src/evals/dataset.py

@@ -1,24 +0,0 @@
-import csv
-import json
-from typing import Any
-
-
-def load_dataset(file_path: str) -> list[dict[str, Any]]:
-    """
-    Loads a dataset from a CSV or JSONL file.
-
-    Args:
-        file_path: The path to the dataset file.
-
-    Returns:
-        A list of dictionaries, where each dictionary represents an example.
-    """
-    if file_path.endswith(".csv"):
-        with open(file_path, encoding="utf-8") as f:
-            reader = csv.DictReader(f)
-            return list(reader)
-    elif file_path.endswith(".jsonl"):
-        with open(file_path, encoding="utf-8") as f:
-            return [json.loads(line) for line in f]
-    else:
-        raise ValueError("Unsupported file format. Please use CSV or JSONL.")

universal_mcp-0.1.24rc13/src/evals/datasets/creative_writing.jsonl

@@ -1,3 +0,0 @@
-{"user_input": "Write a haiku about a rainy day."}
-{"user_input": "Tell me a short story about a friendly ghost."}
-{"user_input": "Write a poem about the changing seasons."}

universal_mcp-0.1.24rc13/src/evals/datasets/exact.jsonl

@@ -1,6 +0,0 @@
-{"user_input": "What is 2 + 2?", "expected_output": "4"}
-{"user_input": "What is the opposite of 'hot'?", "expected_output": "cold"}
-{"user_input": "Who is the first president of the United States?", "expected_output": "George Washington"}
-{"user_input": "What is the capital of France?", "expected_output": "Paris"}
-{"user_input": "Who wrote 'To Kill a Mockingbird'?", "expected_output": "Harper Lee"}
-{"user_input": "What is the boiling point of water at sea level in Celsius?", "expected_output": "100"}

universal_mcp-0.1.24rc13/src/evals/datasets/tasks.jsonl

@@ -1,22 +0,0 @@
-{"user_input": "Send an email to manoj@agentr.dev from my Gmail account", "difficulty": 1, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["send_email"]}}}
-{"user_input": "Show me events from today's Google Calendar.", "difficulty": 1, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["list_events"]}}}}
-{"user_input": "Create a Google Doc summarizing the last 5 merged pull requests in my GitHub repo- universal-mcp/universal-mcp, including links and commit highlights.", "difficulty": 4, "expected_tools": {"agentrServers": {"github": {"tools": ["get_pull_request"]}, "google-docs": {"tools": ["create_document"]}}}}
-{"user_input": "Summarize the key insights from all marketing emails received this week from my Gmail and add a section in a Google Doc with action points.", "difficulty": 4, "expected_tools": {"agentrServers": {"google-gemini": {"tools": ["prompt_document"]}, "google-docs": {"tools": ["style_text"]}}}}
-{"user_input": "Create a Google Sheet of the best cafes and restaurants near IIT Bombay", "difficulty": 3, "expected_tools": {"agentrServers": {"google-sheet": {"tools": ["create_spreadsheet"]}, "serpapi": {"tools": ["google_maps_search"]}}}}
-{"user_input": "Track the top posts in r/startups over the past 7 days using Reddit and create a trend report on what's being discussed most (e.g., hiring, funding, MVPs) in a Google Doc.", "difficulty": 5, "expected_tools": {"agentrServers": {"reddit": {"tools": ["search"]}, "google-docs": {"tools": ["add_content", "create_footer"]}}}}
-{"user_input": "Find the best restaurants in Goa using perplexity web search", "difficulty": 2, "expected_tools": {"agentrServers": {}}}
-{"user_input": "List the unread emails from the last 24 hours from my Gmail, sorted by sender.", "difficulty": 2, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["list_messages"]}}}}
-{"user_input": "Tell me how many meetings I have tomorrow and when they start from my Google Calendar.", "difficulty": 1, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["get_today_events"]}}}}
-{"user_input": "Create a meeting with aditakarsh@example.com on the topic of the latest trends in AI at 8PM today using Google Calendar.", "difficulty": 2, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["add_an_event"]}}}}
-{"user_input": "What are the topics of my meetings today from Google Calendar and who are the attendees? Give a 1-line context for each attendee using LinkedIn or web search.", "difficulty": 4, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["list_events"]}, "serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}
-{"user_input": "Fetch my last inbox mail from Microsoft Outlook", "difficulty": 1, "expected_tools": {"agentrServers": {"outlook": {"tools": ["user_get_mail_folder"]}}}}
-{"user_input": "Fetch unsubscribe links from my Gmail inbox for promo emails I have received in the last 7 days", "difficulty": 3, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["delete_filters", "list_filters"]}}}}
-{"user_input": "Fetch all unread emails from Gmail and new tickets from ClickUp for me from last night", "difficulty": 4, "expected_tools": {"agentrServers": {"clickup": {"tools": ["tasks_filter_team_tasks"]}}}}
-{"user_input": "Give me a report on the earnings of Oklo using web search, and projections for the company revenue, stock price", "difficulty": 4, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}
-{"user_input": "Create a weekly expense report from my credit card transactions and categorize spending by type (food, transport, entertainment, etc.) in a Google Sheet", "difficulty": 3, "expected_tools": {"agentrServers": {"google-sheet": {"tools": ["create_spreadsheet"]}}}}
-{"user_input": "Generate a comparison table of SaaS tools for project management using web search, including pricing, features, and user ratings in a Google Sheet", "difficulty": 4, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "google-sheet": {"tools": ["add_table", "list_tables"]}}}}
-{"user_input": "Research the top 10 Y Combinator startups from the latest batch using web search and create a report on their industries and funding status in Google Docs", "difficulty": 5, "expected_tools": {"agentrServers": {"google-docs": {"tools": ["create_footer", "create_header"]}}}}
-{"user_input": "Find and summarize the key takeaways from the latest earnings calls of FAANG companies using web search and create a report in Google Docs", "difficulty": 5, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "google-docs": {"tools": ["style_text", "create_footer"]}}}}
-{"user_input": "Draft personalized LinkedIn outreach messages for 10 potential collaborators in the fintech space based on their recent posts using LinkedIn data in a Google Sheet", "difficulty": 5, "expected_tools": {"agentrServers": {}}}
-{"user_input": "Monitor my Twitter mentions and DMs from the past 48 hours and create a response priority list in Google Sheets", "difficulty": 4, "expected_tools": {"agentrServers": {"twitter": {"tools": ["search_stream"]}, "google-sheet": {"tools": ["add_sheet", "add_table"]}}}}
-{"user_input": "Create a content calendar for next month with trending AI/ML topics using web search and optimal posting times based on my audience analytics in Google Sheets", "difficulty": 5, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["add_an_event"]}, "google-sheet": {"tools": ["append_values"]}, "serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}

universal_mcp-0.1.24rc13/src/evals/evaluators.py

@@ -1,40 +0,0 @@
-from langsmith.evaluation import EvaluationResult, run_evaluator
-from langsmith.schemas import Example, Run
-from openevals.llm import create_llm_as_judge
-from openevals.prompts import CORRECTNESS_PROMPT
-
-
-@run_evaluator
-def exact_match_evaluator(run: Run, example: Example | None = None) -> EvaluationResult:
-    """
-    A simple evaluator that checks for exact match between the agent's output
-    and the expected output from the dataset.
-    """
-    if example is None or "expected_output" not in example.outputs:
-        return EvaluationResult(key="exact_match", score=0, comment="No expected output provided.")
-
-    # The agent's response might be in a list of messages
-    agent_response_raw = run.outputs.get("output", "")
-    if isinstance(agent_response_raw, list):
-        # Extract text from the last dictionary in the list
-        agent_response = agent_response_raw[-1].get("text", "").strip() if agent_response_raw else ""
-    else:
-        agent_response = str(agent_response_raw).strip()
-
-    expected_output = example.outputs["expected_output"].strip()
-
-    if agent_response == expected_output:
-        score = 1
-        comment = "Exact match."
-    else:
-        score = 0
-        comment = f"Mismatch: Expected '{expected_output}', but got '{agent_response}'."
-
-    return EvaluationResult(key="exact_match", score=score, comment=comment)
-
-
-correctness_evaluator = create_llm_as_judge(
-    prompt=CORRECTNESS_PROMPT,
-    feedback_key="correctness",
-    model="anthropic:claude-4-sonnet-20250514",
-)

universal_mcp-0.1.24rc13/src/evals/run.py

@@ -1,149 +0,0 @@
-import argparse
-import asyncio
-from typing import Any
-
-from dotenv import load_dotenv
-from langsmith import Client, aevaluate
-from langsmith.evaluation import RunEvaluator
-
-from evals.dataset import load_dataset
-from evals.evaluators import correctness_evaluator, exact_match_evaluator
-from universal_mcp.agentr.registry import AgentrRegistry
-from universal_mcp.agents.auto import AutoAgent
-from universal_mcp.agents.base import BaseAgent
-from universal_mcp.agents.react import ReactAgent
-from universal_mcp.agents.simple import SimpleAgent
-
-load_dotenv()
-
-
-# 1. Agent Factory
-def get_agent(agent_name: str) -> BaseAgent:
-    """
-    Factory function to get an agent instance by name.
-    """
-    common_params = {
-        "instructions": "You are a helpful assistant.",
-        "model": "anthropic/claude-4-sonnet-20250514",
-        "registry": AgentrRegistry() if agent_name != "simple" else None,
-    }
-    if agent_name == "simple":
-        return SimpleAgent(name="simple-agent", **common_params)
-    elif agent_name == "react":
-        return ReactAgent(name="react-agent", **common_params)
-    elif agent_name == "auto":
-        return AutoAgent(name="auto-agent", **common_params)
-    else:
-        raise ValueError(f"Unknown agent: {agent_name}. Available agents: simple, react, auto")
-
-
-# 2. Evaluator Registry
-EVALUATORS: dict[str, Any] = {
-    "llm_as_judge": correctness_evaluator,
-    "exact_match": exact_match_evaluator,
-}
-
-
-def get_evaluator(evaluator_name: str) -> RunEvaluator:
-    """
-    Retrieves an evaluator from the registry.
-    """
-    evaluator = EVALUATORS.get(evaluator_name)
-    if evaluator is None:
-        raise ValueError(f"Unknown evaluator: {evaluator_name}. Available evaluators: {', '.join(EVALUATORS.keys())}")
-    return evaluator
-
-
-# Wrapper to run the agent and format the output consistently
-async def agent_runner(agent: BaseAgent, inputs: dict):
-    """
-    Runs the agent and returns a dictionary with the final output.
-    """
-    result = await agent.invoke(user_input=inputs["user_input"])
-    # Extract the last message content as the final response
-    if isinstance(result, dict) and "messages" in result and result["messages"]:
-        content = result["messages"][-1].content
-        if isinstance(content, str):
-            final_response = content
-        elif isinstance(content, list):
-            # Handle list of content blocks (e.g., from Anthropic)
-            text_parts = []
-            for item in content:
-                if isinstance(item, dict) and item.get("type") == "text":
-                    text_parts.append(item.get("text", ""))
-            final_response = "\n".join(text_parts)
-        else:
-            final_response = str(content)
-    else:
-        final_response = str(result)
-    return {"output": final_response}
-
-
-async def main(agent_name: str, dataset_path: str, evaluator_name: str):
-    """
-    The main function for the evaluation CLI.
-    """
-    print(f"Starting evaluation with agent='{agent_name}', dataset='{dataset_path}', evaluator='{evaluator_name}'")
-
-    # 1. Get the agent and evaluator
-    agent = get_agent(agent_name)
-    evaluator = get_evaluator(evaluator_name)
-
-    # Create a callable for aevaluate
-    async def target_func(inputs: dict):
-        return await agent_runner(agent, inputs)
-
-    # 2. Load the dataset from file
-    dataset_examples = load_dataset(dataset_path)
-
-    # 3. Upload dataset to LangSmith for the evaluation run
-    client = Client()
-    dataset_name = dataset_path.split("/")[-1].split(".")[0]
-    # dataset_name = f"{agent_name}-{evaluator_name}-eval-dataset"
-    try:
-        # If dataset with same name and examples exists, read it.
-        # Otherwise, a new one is created.
-        dataset = client.create_dataset(
-            dataset_name, description=f"Dataset for {agent_name} evaluation with {evaluator_name}."
-        )
-        for example in dataset_examples:
-            client.create_example(
-                inputs={"user_input": example["user_input"]},
-                outputs={"output": example.get("expected_output")} if "expected_output" in example else None,
-                dataset_id=dataset.id,
-            )
-        print(f"Created and populated dataset '{dataset_name}' for this run.")
-    except Exception:
-        print(f"Using existing dataset '{dataset_name}'.")
-
-    # 4. Run the evaluation
-    await aevaluate(
-        target_func,
-        data=dataset_name,  # Pass the dataset name
-        evaluators=[evaluator],
-        experiment_prefix=f"{agent_name}-{evaluator_name}-eval",
-    )
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run evaluations on different agents.")
-    parser.add_argument(
-        "agent",
-        type=str,
-        choices=["simple", "react", "auto"],
-        help="The name of the agent to evaluate.",
-    )
-    parser.add_argument(
-        "dataset",
-        type=str,
-        help="Path to the dataset file (e.g., src/evals/example_dataset.jsonl).",
-    )
-    parser.add_argument(
-        "evaluator",
-        type=str,
-        choices=list(EVALUATORS.keys()),
-        help="The name of the evaluator to use.",
-    )
-    args = parser.parse_args()
-
-    asyncio.run(main(agent_name=args.agent, dataset_path=args.dataset, evaluator_name=args.evaluator))

universal_mcp-0.1.24rc13/src/evals/test.py

@@ -1,41 +0,0 @@
-import asyncio
-
-from dotenv import load_dotenv
-from langsmith import Client, aevaluate
-
-from universal_mcp.agentr.registry import AgentrRegistry
-from universal_mcp.agents.autoagent import AutoAgent
-
-load_dotenv()
-
-
-async def target_function1(inputs: dict):
-    agent = AutoAgent(
-        name="autoagent",
-        instructions="You are a helpful assistant that can use tools to help the user.",
-        model="anthropic/claude-4-sonnet-20250514",
-        tool_registry=AgentrRegistry(),
-    )
-    result = await agent.invoke(inputs["user_input"])
-    return result
-
-
-if __name__ == "__main__":
-    client = Client()
-    dataset_name = "autoagent-actual-runs"
-    experiment = asyncio.run(
-        aevaluate(
-            target_function1,
-            data=client.list_examples(dataset_name=dataset_name, splits=["choose-app"]),
-            evaluators=[],
-            experiment_prefix="test-1",
-        )
-    )
-    asyncio.run(
-        aevaluate(
-            target_function1,
-            data=client.list_examples(dataset_name=dataset_name),
-            evaluators=[],
-            experiment=experiment.experiment_name,
-        )
-    )

universal_mcp-0.1.24rc13/src/evals/utils.py

@@ -1,114 +0,0 @@
-import argparse
-
-from dotenv import load_dotenv
-from langsmith import Client
-
-from evals.dataset import load_dataset
-
-load_dotenv()
-
-
-def upload_runs_to_dataset(project_name: str, dataset_name: str, dataset_description: str = ""):
-    """
-    Uploads runs from a LangSmith project to a dataset.
-
-    Args:
-        project_name: The name of the LangSmith project containing the runs.
-        dataset_name: The name of the dataset to create or add to.
-        dataset_description: A description for the new dataset.
-    """
-    client = Client()
-    try:
-        dataset = client.create_dataset(dataset_name, description=dataset_description)
-        print(f"Created new dataset: '{dataset_name}'")
-    except Exception:
-        dataset = client.read_dataset(dataset_name=dataset_name)
-        print(f"Using existing dataset: '{dataset_name}'")
-
-    runs = client.list_runs(project_name=project_name)
-
-    example_count = 0
-    for run in runs:
-        client.create_example(
-            inputs=run.inputs,
-            outputs=run.outputs,
-            dataset_id=dataset.id,
-        )
-        example_count += 1
-
-    print(f"✅ Successfully uploaded {example_count} runs from project '{project_name}' to dataset '{dataset_name}'.")
-
-
-def upload_dataset_from_file(
-    file_path: str,
-    dataset_name: str,
-    dataset_description: str,
-    input_keys: list[str],
-    output_keys: list[str],
-):
-    """
-    Uploads a dataset from a local file (CSV or JSONL) to LangSmith.
-
-    Args:
-        file_path: The path to the local dataset file.
-        dataset_name: The name for the new dataset in LangSmith.
-        dataset_description: A description for the new dataset.
-        input_keys: A list of column names to be used as inputs.
-        output_keys: A list of column names to be used as outputs.
-    """
-    client = Client()
-    examples = load_dataset(file_path)
-
-    try:
-        dataset = client.create_dataset(dataset_name, description=dataset_description)
-        print(f"Created new dataset: '{dataset_name}'")
-    except Exception:
-        dataset = client.read_dataset(dataset_name=dataset_name)
-        print(f"Using existing dataset: '{dataset_name}'")
-
-    for example in examples:
-        inputs = {key: example[key] for key in input_keys if key in example}
-        outputs = {key: example[key] for key in output_keys if key in example}
-        client.create_example(inputs=inputs, outputs=outputs, dataset_id=dataset.id)
-
-    print(f"✅ Successfully uploaded {len(examples)} examples from '{file_path}' to dataset '{dataset_name}'.")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="LangSmith Dataset Utilities.")
-    subparsers = parser.add_subparsers(dest="command", required=True)
-
-    # Sub-parser for uploading runs from a project
-    parser_runs = subparsers.add_parser("upload-runs", help="Upload runs from a project to a dataset.")
-    parser_runs.add_argument("--project-name", required=True, help="The LangSmith project name.")
-    parser_runs.add_argument("--dataset-name", required=True, help="The target dataset name.")
-    parser_runs.add_argument(
-        "--dataset-description", default="Dataset from project runs.", help="Description for the dataset."
-    )
-
-    # Sub-parser for uploading a dataset from a file
-    parser_file = subparsers.add_parser("upload-file", help="Upload a dataset from a local file.")
-    parser_file.add_argument("--file-path", required=True, help="Path to the local dataset file (CSV or JSONL).")
-    parser_file.add_argument("--dataset-name", required=True, help="The name for the dataset in LangSmith.")
-    parser_file.add_argument(
-        "--dataset-description", default="Dataset uploaded from file.", help="Description for the dataset."
-    )
-    parser_file.add_argument("--input-keys", required=True, help="Comma-separated list of input column names.")
-    parser_file.add_argument("--output-keys", required=True, help="Comma-separated list of output column names.")
-
-    args = parser.parse_args()
-
-    if args.command == "upload-runs":
-        upload_runs_to_dataset(
-            project_name=args.project_name,
-            dataset_name=args.dataset_name,
-            dataset_description=args.dataset_description,
-        )
-    elif args.command == "upload-file":
-        upload_dataset_from_file(
-            file_path=args.file_path,
-            dataset_name=args.dataset_name,
-            dataset_description=args.dataset_description,
-            input_keys=args.input_keys.split(","),
-            output_keys=args.output_keys.split(","),
-        )