universal-mcp 0.1.24rc11__tar.gz → 0.1.24rc13__tar.gz

{universal_mcp-0.1.24rc11 → universal_mcp-0.1.24rc13}/PKG-INFO

@@ -1,11 +1,12 @@
 Metadata-Version: 2.4
 Name: universal-mcp
-Version: 0.1.24rc11
+Version: 0.1.24rc13
 Summary: Universal MCP acts as a middle ware for your API applications. It can store your credentials, authorize, enable disable apps on the fly and much more.
 Author-email: Manoj Bajaj <manojbajaj95@gmail.com>
 License: MIT
 License-File: LICENSE
 Requires-Python: >=3.11
+Requires-Dist: agentevals>=0.0.9
 Requires-Dist: black>=25.1.0
 Requires-Dist: cookiecutter>=2.6.0
 Requires-Dist: gql>=4.0.0
@@ -14,6 +15,7 @@ Requires-Dist: jinja2>=3.1.3
 Requires-Dist: jsonref>=1.1.0
 Requires-Dist: keyring>=25.6.0
 Requires-Dist: langchain-anthropic>=0.3.19
+Requires-Dist: langchain-google-genai>=2.1.10
 Requires-Dist: langchain-google-vertexai>=2.0.28
 Requires-Dist: langchain-mcp-adapters>=0.1.9
 Requires-Dist: langchain-openai>=0.3.27
@@ -22,6 +24,7 @@ Requires-Dist: langgraph>=0.5.2
 Requires-Dist: langsmith>=0.4.5
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: mcp>=1.10.0
+Requires-Dist: openevals>=0.1.0
 Requires-Dist: posthog>=3.24.0
 Requires-Dist: pydantic-settings>=2.8.1
 Requires-Dist: pydantic>=2.11.1

{universal_mcp-0.1.24rc11 → universal_mcp-0.1.24rc13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "universal-mcp"
-version = "0.1.24-rc11"
+version = "0.1.24-rc13"
 description = "Universal MCP acts as a middle ware for your API applications. It can store your credentials, authorize, enable disable apps on the fly and much more."
 readme = "README.md"
 authors = [
@@ -14,6 +14,7 @@ requires-python = ">=3.11"
 license = { text = "MIT" }
 dependencies = [
     "Jinja2>=3.1.3",
+    "agentevals>=0.0.9",
     "black>=25.1.0",
     "cookiecutter>=2.6.0",
     "gql>=4.0.0",
@@ -21,6 +22,7 @@ dependencies = [
     "jsonref>=1.1.0",
     "keyring>=25.6.0",
     "langchain-anthropic>=0.3.19",
+    "langchain-google-genai>=2.1.10",
     "langchain-google-vertexai>=2.0.28",
     "langchain-mcp-adapters>=0.1.9",
     "langchain-openai>=0.3.27",
@@ -29,6 +31,7 @@ dependencies = [
     "langsmith>=0.4.5",
     "loguru>=0.7.3",
     "mcp>=1.10.0",
+    "openevals>=0.1.0",
     "posthog>=3.24.0",
     "pydantic>=2.11.1",
     "pydantic-settings>=2.8.1",

universal_mcp-0.1.24rc13/src/evals/dataset.py

@@ -0,0 +1,24 @@
+import csv
+import json
+from typing import Any
+
+
+def load_dataset(file_path: str) -> list[dict[str, Any]]:
+    """
+    Loads a dataset from a CSV or JSONL file.
+
+    Args:
+        file_path: The path to the dataset file.
+
+    Returns:
+        A list of dictionaries, where each dictionary represents an example.
+    """
+    if file_path.endswith(".csv"):
+        with open(file_path, encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            return list(reader)
+    elif file_path.endswith(".jsonl"):
+        with open(file_path, encoding="utf-8") as f:
+            return [json.loads(line) for line in f]
+    else:
+        raise ValueError("Unsupported file format. Please use CSV or JSONL.")

universal_mcp-0.1.24rc13/src/evals/datasets/creative_writing.jsonl

@@ -0,0 +1,3 @@
+{"user_input": "Write a haiku about a rainy day."}
+{"user_input": "Tell me a short story about a friendly ghost."}
+{"user_input": "Write a poem about the changing seasons."}

universal_mcp-0.1.24rc13/src/evals/datasets/exact.jsonl

@@ -0,0 +1,6 @@
+{"user_input": "What is 2 + 2?", "expected_output": "4"}
+{"user_input": "What is the opposite of 'hot'?", "expected_output": "cold"}
+{"user_input": "Who is the first president of the United States?", "expected_output": "George Washington"}
+{"user_input": "What is the capital of France?", "expected_output": "Paris"}
+{"user_input": "Who wrote 'To Kill a Mockingbird'?", "expected_output": "Harper Lee"}
+{"user_input": "What is the boiling point of water at sea level in Celsius?", "expected_output": "100"}

universal_mcp-0.1.24rc13/src/evals/datasets/tasks.jsonl

@@ -0,0 +1,22 @@
+{"user_input": "Send an email to manoj@agentr.dev from my Gmail account", "difficulty": 1, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["send_email"]}}}
+{"user_input": "Show me events from today's Google Calendar.", "difficulty": 1, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["list_events"]}}}}
+{"user_input": "Create a Google Doc summarizing the last 5 merged pull requests in my GitHub repo- universal-mcp/universal-mcp, including links and commit highlights.", "difficulty": 4, "expected_tools": {"agentrServers": {"github": {"tools": ["get_pull_request"]}, "google-docs": {"tools": ["create_document"]}}}}
+{"user_input": "Summarize the key insights from all marketing emails received this week from my Gmail and add a section in a Google Doc with action points.", "difficulty": 4, "expected_tools": {"agentrServers": {"google-gemini": {"tools": ["prompt_document"]}, "google-docs": {"tools": ["style_text"]}}}}
+{"user_input": "Create a Google Sheet of the best cafes and restaurants near IIT Bombay", "difficulty": 3, "expected_tools": {"agentrServers": {"google-sheet": {"tools": ["create_spreadsheet"]}, "serpapi": {"tools": ["google_maps_search"]}}}}
+{"user_input": "Track the top posts in r/startups over the past 7 days using Reddit and create a trend report on what's being discussed most (e.g., hiring, funding, MVPs) in a Google Doc.", "difficulty": 5, "expected_tools": {"agentrServers": {"reddit": {"tools": ["search"]}, "google-docs": {"tools": ["add_content", "create_footer"]}}}}
+{"user_input": "Find the best restaurants in Goa using perplexity web search", "difficulty": 2, "expected_tools": {"agentrServers": {}}}
+{"user_input": "List the unread emails from the last 24 hours from my Gmail, sorted by sender.", "difficulty": 2, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["list_messages"]}}}}
+{"user_input": "Tell me how many meetings I have tomorrow and when they start from my Google Calendar.", "difficulty": 1, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["get_today_events"]}}}}
+{"user_input": "Create a meeting with aditakarsh@example.com on the topic of the latest trends in AI at 8PM today using Google Calendar.", "difficulty": 2, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["add_an_event"]}}}}
+{"user_input": "What are the topics of my meetings today from Google Calendar and who are the attendees? Give a 1-line context for each attendee using LinkedIn or web search.", "difficulty": 4, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["list_events"]}, "serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}
+{"user_input": "Fetch my last inbox mail from Microsoft Outlook", "difficulty": 1, "expected_tools": {"agentrServers": {"outlook": {"tools": ["user_get_mail_folder"]}}}}
+{"user_input": "Fetch unsubscribe links from my Gmail inbox for promo emails I have received in the last 7 days", "difficulty": 3, "expected_tools": {"agentrServers": {"google-mail": {"tools": ["delete_filters", "list_filters"]}}}}
+{"user_input": "Fetch all unread emails from Gmail and new tickets from ClickUp for me from last night", "difficulty": 4, "expected_tools": {"agentrServers": {"clickup": {"tools": ["tasks_filter_team_tasks"]}}}}
+{"user_input": "Give me a report on the earnings of Oklo using web search, and projections for the company revenue, stock price", "difficulty": 4, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}
+{"user_input": "Create a weekly expense report from my credit card transactions and categorize spending by type (food, transport, entertainment, etc.) in a Google Sheet", "difficulty": 3, "expected_tools": {"agentrServers": {"google-sheet": {"tools": ["create_spreadsheet"]}}}}
+{"user_input": "Generate a comparison table of SaaS tools for project management using web search, including pricing, features, and user ratings in a Google Sheet", "difficulty": 4, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "google-sheet": {"tools": ["add_table", "list_tables"]}}}}
+{"user_input": "Research the top 10 Y Combinator startups from the latest batch using web search and create a report on their industries and funding status in Google Docs", "difficulty": 5, "expected_tools": {"agentrServers": {"google-docs": {"tools": ["create_footer", "create_header"]}}}}
+{"user_input": "Find and summarize the key takeaways from the latest earnings calls of FAANG companies using web search and create a report in Google Docs", "difficulty": 5, "expected_tools": {"agentrServers": {"serpapi": {"tools": ["search"]}, "google-docs": {"tools": ["style_text", "create_footer"]}}}}
+{"user_input": "Draft personalized LinkedIn outreach messages for 10 potential collaborators in the fintech space based on their recent posts using LinkedIn data in a Google Sheet", "difficulty": 5, "expected_tools": {"agentrServers": {}}}
+{"user_input": "Monitor my Twitter mentions and DMs from the past 48 hours and create a response priority list in Google Sheets", "difficulty": 4, "expected_tools": {"agentrServers": {"twitter": {"tools": ["search_stream"]}, "google-sheet": {"tools": ["add_sheet", "add_table"]}}}}
+{"user_input": "Create a content calendar for next month with trending AI/ML topics using web search and optimal posting times based on my audience analytics in Google Sheets", "difficulty": 5, "expected_tools": {"agentrServers": {"google-calendar": {"tools": ["add_an_event"]}, "google-sheet": {"tools": ["append_values"]}, "serpapi": {"tools": ["search"]}, "perplexity": {"tools": ["chat"]}, "tavily": {"tools": ["search"]}}}}

universal_mcp-0.1.24rc13/src/evals/evaluators.py

@@ -0,0 +1,40 @@
+from langsmith.evaluation import EvaluationResult, run_evaluator
+from langsmith.schemas import Example, Run
+from openevals.llm import create_llm_as_judge
+from openevals.prompts import CORRECTNESS_PROMPT
+
+
+@run_evaluator
+def exact_match_evaluator(run: Run, example: Example | None = None) -> EvaluationResult:
+    """
+    A simple evaluator that checks for exact match between the agent's output
+    and the expected output from the dataset.
+    """
+    if example is None or "expected_output" not in example.outputs:
+        return EvaluationResult(key="exact_match", score=0, comment="No expected output provided.")
+
+    # The agent's response might be in a list of messages
+    agent_response_raw = run.outputs.get("output", "")
+    if isinstance(agent_response_raw, list):
+        # Extract text from the last dictionary in the list
+        agent_response = agent_response_raw[-1].get("text", "").strip() if agent_response_raw else ""
+    else:
+        agent_response = str(agent_response_raw).strip()
+
+    expected_output = example.outputs["expected_output"].strip()
+
+    if agent_response == expected_output:
+        score = 1
+        comment = "Exact match."
+    else:
+        score = 0
+        comment = f"Mismatch: Expected '{expected_output}', but got '{agent_response}'."
+
+    return EvaluationResult(key="exact_match", score=score, comment=comment)
+
+
+correctness_evaluator = create_llm_as_judge(
+    prompt=CORRECTNESS_PROMPT,
+    feedback_key="correctness",
+    model="anthropic:claude-4-sonnet-20250514",
+)

universal_mcp-0.1.24rc13/src/evals/run.py

@@ -0,0 +1,149 @@
+import argparse
+import asyncio
+from typing import Any
+
+from dotenv import load_dotenv
+from langsmith import Client, aevaluate
+from langsmith.evaluation import RunEvaluator
+
+from evals.dataset import load_dataset
+from evals.evaluators import correctness_evaluator, exact_match_evaluator
+from universal_mcp.agentr.registry import AgentrRegistry
+from universal_mcp.agents.auto import AutoAgent
+from universal_mcp.agents.base import BaseAgent
+from universal_mcp.agents.react import ReactAgent
+from universal_mcp.agents.simple import SimpleAgent
+
+load_dotenv()
+
+
+# 1. Agent Factory
+def get_agent(agent_name: str) -> BaseAgent:
+    """
+    Factory function to get an agent instance by name.
+    """
+    common_params = {
+        "instructions": "You are a helpful assistant.",
+        "model": "anthropic/claude-4-sonnet-20250514",
+        "registry": AgentrRegistry() if agent_name != "simple" else None,
+    }
+    if agent_name == "simple":
+        return SimpleAgent(name="simple-agent", **common_params)
+    elif agent_name == "react":
+        return ReactAgent(name="react-agent", **common_params)
+    elif agent_name == "auto":
+        return AutoAgent(name="auto-agent", **common_params)
+    else:
+        raise ValueError(f"Unknown agent: {agent_name}. Available agents: simple, react, auto")
+
+
+# 2. Evaluator Registry
+EVALUATORS: dict[str, Any] = {
+    "llm_as_judge": correctness_evaluator,
+    "exact_match": exact_match_evaluator,
+}
+
+
+def get_evaluator(evaluator_name: str) -> RunEvaluator:
+    """
+    Retrieves an evaluator from the registry.
+    """
+    evaluator = EVALUATORS.get(evaluator_name)
+    if evaluator is None:
+        raise ValueError(f"Unknown evaluator: {evaluator_name}. Available evaluators: {', '.join(EVALUATORS.keys())}")
+    return evaluator
+
+
+# Wrapper to run the agent and format the output consistently
+async def agent_runner(agent: BaseAgent, inputs: dict):
+    """
+    Runs the agent and returns a dictionary with the final output.
+    """
+    result = await agent.invoke(user_input=inputs["user_input"])
+    # Extract the last message content as the final response
+    if isinstance(result, dict) and "messages" in result and result["messages"]:
+        content = result["messages"][-1].content
+        if isinstance(content, str):
+            final_response = content
+        elif isinstance(content, list):
+            # Handle list of content blocks (e.g., from Anthropic)
+            text_parts = []
+            for item in content:
+                if isinstance(item, dict) and item.get("type") == "text":
+                    text_parts.append(item.get("text", ""))
+            final_response = "\n".join(text_parts)
+        else:
+            final_response = str(content)
+    else:
+        final_response = str(result)
+    return {"output": final_response}
+
+
+async def main(agent_name: str, dataset_path: str, evaluator_name: str):
+    """
+    The main function for the evaluation CLI.
+    """
+    print(f"Starting evaluation with agent='{agent_name}', dataset='{dataset_path}', evaluator='{evaluator_name}'")
+
+    # 1. Get the agent and evaluator
+    agent = get_agent(agent_name)
+    evaluator = get_evaluator(evaluator_name)
+
+    # Create a callable for aevaluate
+    async def target_func(inputs: dict):
+        return await agent_runner(agent, inputs)
+
+    # 2. Load the dataset from file
+    dataset_examples = load_dataset(dataset_path)
+
+    # 3. Upload dataset to LangSmith for the evaluation run
+    client = Client()
+    dataset_name = dataset_path.split("/")[-1].split(".")[0]
+    # dataset_name = f"{agent_name}-{evaluator_name}-eval-dataset"
+    try:
+        # If dataset with same name and examples exists, read it.
+        # Otherwise, a new one is created.
+        dataset = client.create_dataset(
+            dataset_name, description=f"Dataset for {agent_name} evaluation with {evaluator_name}."
+        )
+        for example in dataset_examples:
+            client.create_example(
+                inputs={"user_input": example["user_input"]},
+                outputs={"output": example.get("expected_output")} if "expected_output" in example else None,
+                dataset_id=dataset.id,
+            )
+        print(f"Created and populated dataset '{dataset_name}' for this run.")
+    except Exception:
+        print(f"Using existing dataset '{dataset_name}'.")
+
+    # 4. Run the evaluation
+    await aevaluate(
+        target_func,
+        data=dataset_name,  # Pass the dataset name
+        evaluators=[evaluator],
+        experiment_prefix=f"{agent_name}-{evaluator_name}-eval",
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run evaluations on different agents.")
+    parser.add_argument(
+        "agent",
+        type=str,
+        choices=["simple", "react", "auto"],
+        help="The name of the agent to evaluate.",
+    )
+    parser.add_argument(
+        "dataset",
+        type=str,
+        help="Path to the dataset file (e.g., src/evals/example_dataset.jsonl).",
+    )
+    parser.add_argument(
+        "evaluator",
+        type=str,
+        choices=list(EVALUATORS.keys()),
+        help="The name of the evaluator to use.",
+    )
+    args = parser.parse_args()
+
+    asyncio.run(main(agent_name=args.agent, dataset_path=args.dataset, evaluator_name=args.evaluator))

universal_mcp-0.1.24rc13/src/evals/test.py

@@ -0,0 +1,41 @@
+import asyncio
+
+from dotenv import load_dotenv
+from langsmith import Client, aevaluate
+
+from universal_mcp.agentr.registry import AgentrRegistry
+from universal_mcp.agents.autoagent import AutoAgent
+
+load_dotenv()
+
+
+async def target_function1(inputs: dict):
+    agent = AutoAgent(
+        name="autoagent",
+        instructions="You are a helpful assistant that can use tools to help the user.",
+        model="anthropic/claude-4-sonnet-20250514",
+        tool_registry=AgentrRegistry(),
+    )
+    result = await agent.invoke(inputs["user_input"])
+    return result
+
+
+if __name__ == "__main__":
+    client = Client()
+    dataset_name = "autoagent-actual-runs"
+    experiment = asyncio.run(
+        aevaluate(
+            target_function1,
+            data=client.list_examples(dataset_name=dataset_name, splits=["choose-app"]),
+            evaluators=[],
+            experiment_prefix="test-1",
+        )
+    )
+    asyncio.run(
+        aevaluate(
+            target_function1,
+            data=client.list_examples(dataset_name=dataset_name),
+            evaluators=[],
+            experiment=experiment.experiment_name,
+        )
+    )

universal_mcp-0.1.24rc13/src/evals/utils.py

@@ -0,0 +1,114 @@
+import argparse
+
+from dotenv import load_dotenv
+from langsmith import Client
+
+from evals.dataset import load_dataset
+
+load_dotenv()
+
+
+def upload_runs_to_dataset(project_name: str, dataset_name: str, dataset_description: str = ""):
+    """
+    Uploads runs from a LangSmith project to a dataset.
+
+    Args:
+        project_name: The name of the LangSmith project containing the runs.
+        dataset_name: The name of the dataset to create or add to.
+        dataset_description: A description for the new dataset.
+    """
+    client = Client()
+    try:
+        dataset = client.create_dataset(dataset_name, description=dataset_description)
+        print(f"Created new dataset: '{dataset_name}'")
+    except Exception:
+        dataset = client.read_dataset(dataset_name=dataset_name)
+        print(f"Using existing dataset: '{dataset_name}'")
+
+    runs = client.list_runs(project_name=project_name)
+
+    example_count = 0
+    for run in runs:
+        client.create_example(
+            inputs=run.inputs,
+            outputs=run.outputs,
+            dataset_id=dataset.id,
+        )
+        example_count += 1
+
+    print(f"✅ Successfully uploaded {example_count} runs from project '{project_name}' to dataset '{dataset_name}'.")
+
+
+def upload_dataset_from_file(
+    file_path: str,
+    dataset_name: str,
+    dataset_description: str,
+    input_keys: list[str],
+    output_keys: list[str],
+):
+    """
+    Uploads a dataset from a local file (CSV or JSONL) to LangSmith.
+
+    Args:
+        file_path: The path to the local dataset file.
+        dataset_name: The name for the new dataset in LangSmith.
+        dataset_description: A description for the new dataset.
+        input_keys: A list of column names to be used as inputs.
+        output_keys: A list of column names to be used as outputs.
+    """
+    client = Client()
+    examples = load_dataset(file_path)
+
+    try:
+        dataset = client.create_dataset(dataset_name, description=dataset_description)
+        print(f"Created new dataset: '{dataset_name}'")
+    except Exception:
+        dataset = client.read_dataset(dataset_name=dataset_name)
+        print(f"Using existing dataset: '{dataset_name}'")
+
+    for example in examples:
+        inputs = {key: example[key] for key in input_keys if key in example}
+        outputs = {key: example[key] for key in output_keys if key in example}
+        client.create_example(inputs=inputs, outputs=outputs, dataset_id=dataset.id)
+
+    print(f"✅ Successfully uploaded {len(examples)} examples from '{file_path}' to dataset '{dataset_name}'.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="LangSmith Dataset Utilities.")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # Sub-parser for uploading runs from a project
+    parser_runs = subparsers.add_parser("upload-runs", help="Upload runs from a project to a dataset.")
+    parser_runs.add_argument("--project-name", required=True, help="The LangSmith project name.")
+    parser_runs.add_argument("--dataset-name", required=True, help="The target dataset name.")
+    parser_runs.add_argument(
+        "--dataset-description", default="Dataset from project runs.", help="Description for the dataset."
+    )
+
+    # Sub-parser for uploading a dataset from a file
+    parser_file = subparsers.add_parser("upload-file", help="Upload a dataset from a local file.")
+    parser_file.add_argument("--file-path", required=True, help="Path to the local dataset file (CSV or JSONL).")
+    parser_file.add_argument("--dataset-name", required=True, help="The name for the dataset in LangSmith.")
+    parser_file.add_argument(
+        "--dataset-description", default="Dataset uploaded from file.", help="Description for the dataset."
+    )
+    parser_file.add_argument("--input-keys", required=True, help="Comma-separated list of input column names.")
+    parser_file.add_argument("--output-keys", required=True, help="Comma-separated list of output column names.")
+
+    args = parser.parse_args()
+
+    if args.command == "upload-runs":
+        upload_runs_to_dataset(
+            project_name=args.project_name,
+            dataset_name=args.dataset_name,
+            dataset_description=args.dataset_description,
+        )
+    elif args.command == "upload-file":
+        upload_dataset_from_file(
+            file_path=args.file_path,
+            dataset_name=args.dataset_name,
+            dataset_description=args.dataset_description,
+            input_keys=args.input_keys.split(","),
+            output_keys=args.output_keys.split(","),
+        )