universal-mcp-agents 0.1.13__tar.gz → 0.1.14__tar.gz

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: universal-mcp-agents
-Version: 0.1.13
+Version: 0.1.14
 Summary: Add your description here
 Project-URL: Homepage, https://github.com/universal-mcp/applications
 Project-URL: Repository, https://github.com/universal-mcp/applications

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "universal-mcp-agents"
-version = "0.1.13"
+version = "0.1.14"
 description = "Add your description here"
 readme = "README.md"
 authors = [

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/evals/datasets/tasks.jsonl

@@ -10,7 +10,7 @@
 {"user_input": "search reddit for posts on elon musk and then post a meme on him on linkedin", "difficulty": 3, "required_tools": {"reddit" : ["search_reddit"], "linkedin": ["create_post"]}}
 {"user_input": "Search for best cafes near IIT bombay using exa and make a google sheet out of it", "difficulty": 3, "required_tools": {"exa": ["search_with_filters"], "google_sheet": ["create_spreadsheet", "write_values_to_sheet", "add_table"]}}
 {"user_input": "Create a Google Doc summarizing the last 5 merged pull requests in my GitHub repo- universal-mcp/universal-mcp, including links and commit highlights.", "difficulty": 4, "required_tools": {"github": ["list_pull_requests", "list_recent_commits"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
-{"user_input": "Summarize the key insights from all marketing emails received yesterday from my Gmail and add a section in a Google Doc with action points.", "difficulty": 4, "required_tools": {"google_mail": ["list_messages"], "google_docs": ["create_document"]}}
+{"user_input": "Summarize the key insights from all marketing emails received yesterday from my Gmail and add a section in a Google Doc with action points.", "difficulty": 4, "required_tools": {"google_mail": ["list_messages"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
 {"user_input": "Give me a report on the earnings of Oklo using web search, and projections for the company revenue, stock price", "difficulty": 4, "required_tools": {"tavily": ["search_and_summarize"]}}
 {"user_input": "Track the top posts in r/startups over the past 7 days using Reddit and create a trend report on what's being discussed most (e.g., hiring, funding, MVPs) in a Google Doc.", "difficulty": 4, "required_tools": {"reddit": ["get_subreddit_posts", "get_subreddit_top_posts"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
 {"user_input": "Generate a comparison table of SaaS tools for project management using web search, including pricing, features, and user ratings in a Google Sheet", "difficulty": 4, "required_tools": {"tavily": ["search_and_summarize"], "google_sheet": ["create_spreadsheet", "add_table"]}}

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/evals/evaluators.py

@@ -7,6 +7,8 @@ from langsmith.evaluation import EvaluationResult, run_evaluator
 from langsmith.schemas import Example, Run
 from openevals.llm import create_llm_as_judge
 
+from evals.prompts import CODEACT_EVALUATOR_PROMPT, CORRECTNESS_PROMPT
+
 
 @run_evaluator
 def exact_match_evaluator(run: Run, example: Example | None = None) -> EvaluationResult:
@@ -38,58 +40,6 @@ def exact_match_evaluator(run: Run, example: Example | None = None) -> Evaluatio
     return EvaluationResult(key="exact_match", score=score, comment=comment)
 
 
-CORRECTNESS_PROMPT = """You are an expert data labeler evaluating model outputs for correctness. Your task is to assign a score based on the following rubric:
-
-<Rubric>
-  A correct answer:
-  - Provides accurate and complete information
-  - Contains no factual errors
-  - Addresses all parts of the question
-  - Is logically consistent
-  - Uses precise and accurate terminology
-
-  When scoring, you should penalize:
-  - Factual errors or inaccuracies
-  - Incomplete or partial answers
-  - Misleading or ambiguous statements
-  - Incorrect terminology
-  - Logical inconsistencies
-  - Missing key information
-
-  Ignore the following:
-  - If the answer is not in the same language as the question.
-  - use the specifically requested tool, as the tool name can be different
-  - Do not penalize for incorrect third party data coming from the tool.
-</Rubric>
-
-<Instructions>
-  - Carefully read the input and output
-  - Check for factual accuracy and completeness
-  - Focus on correctness of information rather than style or verbosity
-  - If the user tool is not authorized, give a partial credit of `0.5`
-  - Give partial credit if tools and called correctly, but the data is incorrect from tools.
-</Instructions>
-
-<Reminder>
-  The goal is to evaluate factual correctness and completeness of the response.
-</Reminder>
-
-<input>
-{inputs}
-</input>
-
-<output>
-{outputs}
-</output>
-
-Use the reference outputs below to help you evaluate the correctness of the response:
-
-<reference_outputs>
-{reference_outputs}
-</reference_outputs>
-"""
-
-
 correctness_evaluator = create_llm_as_judge(
     prompt=CORRECTNESS_PROMPT,
     feedback_key="correctness",
@@ -103,8 +53,8 @@ trajectory_evaluator = create_trajectory_llm_as_judge(
 )
 
 
-codeact_evaluator = create_trajectory_llm_as_judge(
-    prompt=TRAJECTORY_ACCURACY_PROMPT,
+codeact_evaluator = create_llm_as_judge(
+    prompt=CODEACT_EVALUATOR_PROMPT,
     feedback_key="codeact_accuracy",
     model="anthropic:claude-4-sonnet-20250514",
 )

universal_mcp_agents-0.1.14/src/evals/prompts.py

@@ -0,0 +1,66 @@
+CORRECTNESS_PROMPT = """You are an expert data labeler evaluating model outputs for correctness. Your task is to assign a score based on the following rubric:
+
+<Rubric>
+  A correct answer:
+  - Provides accurate and complete information
+  - Contains no factual errors
+  - Addresses all parts of the question
+  - Is logically consistent
+  - Uses precise and accurate terminology
+
+  When scoring, you should penalize:
+  - Factual errors or inaccuracies
+  - Incomplete or partial answers
+  - Misleading or ambiguous statements
+  - Incorrect terminology
+  - Logical inconsistencies
+  - Missing key information
+
+  Ignore the following:
+  - If the answer is not in the same language as the question.
+  - use the specifically requested tool, as the tool name can be different
+  - Do not penalize for incorrect third party data coming from the tool.
+</Rubric>
+
+<Instructions>
+  - Carefully read the input and output
+  - Check for factual accuracy and completeness
+  - Focus on correctness of information rather than style or verbosity
+  - If the user tool is not authorized, give a partial credit of `0.5`
+  - Give partial credit if tools and called correctly, but the data is incorrect from tools.
+</Instructions>
+
+<Reminder>
+  The goal is to evaluate factual correctness and completeness of the response.
+</Reminder>
+
+<input>
+{inputs}
+</input>
+
+<output>
+{outputs}
+</output>
+
+Use the reference outputs below to help you evaluate the correctness of the response:
+
+<reference_outputs>
+{reference_outputs}
+</reference_outputs>
+"""
+
+CODEACT_EVALUATOR_PROMPT = """
+You are a code execution evaluator. You will be given the entire run of an agent, starting with a human input task, the intermediate steps taken, and the final output of the agent given to the user. These steps will contain code written by the agent to solve the problem as well as its outputs. Your job is to check ONLY if the code executes correctly.
+Keep in mind that the agent has access to tools like- ai_classify, call_llm, creative_writer, data_extractor. These calls are to be treated as valid if they run without errors.
+These are the only criteria you should evaluate-
+ 
+<Rubric>
+- The code written by the agent in tool calls should be syntactically correct and use existing objects.
+- The code outputs should not have an error or empty/unexpected outputs
+</Rubric>
+If either of the above are not satisfied, you should give 0.
+ 
+<Reminder>
+You must not judge whether the code is helpful to the task or not, only if the code itself is correct or not.
+</Reminder>
+"""

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/evals/run.py

@@ -15,6 +15,7 @@ from evals.evaluators import (
     exact_match_evaluator,
     tool_node_evaluator,
     trajectory_evaluator,
+    codeact_evaluator,
 )
 from universal_mcp.agents import get_agent
 from universal_mcp.agents.utils import messages_to_list
@@ -62,12 +63,12 @@ async def agent_runner(agent_name: str, inputs: dict) -> dict:
     registry = AgentrRegistry(client=client) if agent_name != "simple" else None
     common_params = {
         "instructions": f"You are a helpful assistant. Keep your responses short and concise. Do not provide with any explanation. The current date and time is {current_date_time}",
-        "model": "anthropic/claude-4-sonnet-20250514",
+        "model": "azure/gpt-4.1",
         "registry": registry,
         "tools": inputs.get("tools", {}),
     }
     agent = get_agent(agent_name)(name=agent_name, **common_params)
-    result = await agent.invoke(user_input=inputs["user_input"])
+    result = await agent.invoke(user_input=inputs["user_input"], thread_id="evals")
     messages = messages_to_list(result["messages"])
     return_result = {"output": messages}
     if "tool_config" in result:

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/tests/test_agents.py

@@ -145,6 +145,7 @@ class MockToolRegistry(ToolRegistry):
         self,
         query: str,
         limit: int = 10,
+        **kwargs: Any,
     ) -> list[dict[str, Any]]:
         """
         Search for apps by a query.
@@ -167,6 +168,7 @@ class MockToolRegistry(ToolRegistry):
         query: str,
         limit: int = 10,
         app_id: str | None = None,
+        **kwargs: Any,
     ) -> list[dict[str, Any]]:
         """
         Search for tools by a query.

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/universal_mcp/agents/base.py

@@ -115,6 +115,8 @@ class BaseAgent:
             "recursion_limit": 25,
             "configurable": {"thread_id": thread_id},
             "metadata": run_metadata,
+            "run_id": thread_id,
+            "run_name" : self.name
         }
 
         result = await self._graph.ainvoke(

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/universal_mcp/agents/bigtool/__init__.py

@@ -56,7 +56,7 @@ class BigToolAgent(BaseAgent):
             compiled_graph = graph_builder.compile(checkpointer=self.memory)
             return compiled_graph
         except Exception as e:
-            raise e
+            raise Exception(f"Failed to build AutoAgent graph: {e}")
 
     @property
     def graph(self):

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/universal_mcp/agents/bigtool/agent.py

@@ -1,9 +1,9 @@
 from universal_mcp.agentr.registry import AgentrRegistry
-from universal_mcp.agents.bigtoolcache import BigToolAgentCache
+from universal_mcp.agents.bigtool import BigToolAgent
 
 
 async def agent():
-    agent_object = await BigToolAgentCache(
+    agent_object = await BigToolAgent(
         registry=AgentrRegistry(),
     )._build_graph()
     return agent_object

universal_mcp_agents-0.1.14/src/universal_mcp/agents/bigtool/graph.py

@@ -0,0 +1,149 @@
+import json
+from typing import Literal, cast
+
+from dotenv import load_dotenv
+from langchain_anthropic import ChatAnthropic
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import AIMessage, SystemMessage, ToolMessage
+from langchain_core.tools import BaseTool
+from langgraph.graph import StateGraph
+from langgraph.types import Command, RetryPolicy
+from universal_mcp.tools.registry import ToolRegistry
+from universal_mcp.types import ToolFormat
+
+from .state import State
+from .tools import get_valid_tools
+from universal_mcp.agents.utils import filter_retry_on
+
+load_dotenv()
+
+
+def build_graph(
+    registry: ToolRegistry,
+    base_model: BaseChatModel,
+    system_prompt: str,
+    default_tools: list[BaseTool],
+    meta_tools: dict[str, BaseTool],
+):
+    """Build the LangGraph workflow"""
+
+    async def agent_node(state: State) -> Command[Literal["execute_tools"]]:
+        """Main agent reasoning node"""
+
+        # Combine meta tools with currently loaded tools
+        if len(state["selected_tool_ids"]) > 0:
+            try:
+                current_tools = await registry.export_tools(
+                    tools=state["selected_tool_ids"], format=ToolFormat.LANGCHAIN
+                )
+            except Exception as e:
+                raise Exception(f"Failed to export selected tools: {e}")
+        else:
+            current_tools = []
+        all_tools = (
+            [meta_tools["search_tools"], meta_tools["load_tools"], meta_tools.get("web_search")]
+            + default_tools
+            + current_tools
+        )
+
+        # Remove duplicates based on tool name
+        seen_names = set()
+        unique_tools = []
+        for tool in all_tools:
+            if tool.name not in seen_names:
+                seen_names.add(tool.name)
+                unique_tools.append(tool)
+
+        try:
+            if isinstance(base_model, ChatAnthropic):
+                model_with_tools = base_model.bind_tools(
+                    unique_tools,
+                    tool_choice="auto",
+                    parallel_tool_calls=False,
+                    cache_control={"type": "ephemeral", "ttl": "1h"},
+                )
+            else:
+                model_with_tools = base_model.bind_tools(
+                    unique_tools,
+                    tool_choice="auto",
+                    parallel_tool_calls=False,
+                )
+        except Exception as e:
+            raise Exception(f"Failed to bind tools to model: {e}")
+
+        # Get response from model
+        messages = [SystemMessage(content=system_prompt), *state["messages"]]
+
+        try:
+            response = cast(AIMessage, await model_with_tools.ainvoke(messages))
+        except Exception as e:
+            raise Exception(f"Model invocation failed: {e}")
+
+        if response.tool_calls:
+            return Command(goto="execute_tools", update={"messages": [response]})
+        else:
+            return Command(update={"messages": [response], "model_with_tools": model_with_tools})
+
+    async def execute_tools_node(state: State) -> Command[Literal["agent"]]:
+        """Execute tool calls"""
+        last_message = state["messages"][-1]
+        tool_calls = last_message.tool_calls if isinstance(last_message, AIMessage) else []
+
+        tool_messages = []
+        new_tool_ids = []
+        ask_user = False
+
+        for tool_call in tool_calls:
+            try:
+                if tool_call["name"] == "load_tools":  # Handle load_tools separately
+                    valid_tools, unconnected_links = await get_valid_tools(tool_ids=tool_call["args"]["tool_ids"], registry=registry)
+                    new_tool_ids.extend(valid_tools)
+                    # Create tool message response
+                    tool_result = f"Successfully loaded {len(valid_tools)} tools: {valid_tools}"
+                    if unconnected_links:
+                        ask_user = True
+                        ai_msg = f"Please login to the following app(s) using the following links and let me know in order to proceed:\n {'\n'.join(unconnected_links)} "
+
+                elif tool_call["name"] == "search_tools":
+                    tool_result = await meta_tools["search_tools"].ainvoke(tool_call["args"])
+                elif tool_call["name"] == "web_search":
+                    tool_result = await meta_tools["web_search"].ainvoke(tool_call["args"])
+                else:
+                    # Load tools first
+                    await registry.export_tools([tool_call["name"]], ToolFormat.LANGCHAIN)
+                    tool_result = await registry.call_tool(tool_call["name"], tool_call["args"])
+            except Exception as e:
+                tool_result = f"Error during {tool_call}: {e}"
+
+
+            tool_message = ToolMessage(
+                content=json.dumps(tool_result),
+                name=tool_call["name"],
+                tool_call_id=tool_call["id"],
+            )
+            tool_messages.append(tool_message)
+        if ask_user:
+            tool_messages.append(AIMessage(content=ai_msg))
+            return Command(update={"messages": tool_messages, "selected_tool_ids": new_tool_ids})
+
+        return Command(goto="agent", update={"messages": tool_messages, "selected_tool_ids": new_tool_ids})
+
+    # Define the graph
+    workflow = StateGraph(State)
+
+    # Add nodes
+    workflow.add_node(
+        "agent",
+        agent_node,
+        retry_policy=RetryPolicy(max_attempts=3, retry_on=filter_retry_on, initial_interval=2, backoff_factor=2),
+    )
+    workflow.add_node(
+        "execute_tools",
+        execute_tools_node,
+        retry_policy=RetryPolicy(max_attempts=3, retry_on=filter_retry_on, initial_interval=2, backoff_factor=2),
+    )
+
+    # Set entry point
+    workflow.set_entry_point("agent")
+
+    return workflow

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/universal_mcp/agents/bigtool/prompts.py

@@ -5,9 +5,9 @@ SYSTEM_PROMPT = """You are a helpful AI assistant, called {name}.
 **Core Directives:**
 1.  **Always Use Tools for Tasks:** For any user request that requires an action (e.g., sending an email, searching for information, creating an event, displaying a chart), you MUST use a tool. Do not refuse a task if a tool might exist for it.
 
-2. Check if your existing tools or knowledge can handle the user's request. If they can, use them. If they cannot, you must call the `search_tools` function to find the right tools for the user's request.You must not use the same/similar query multiple times in the list. The list should have multiple queries only if the task has clearly different sub-tasks. If you do not find any specific relevant tools, use the pre-loaded generic tools.
+2. Check if your existing tools or knowledge can handle the user's request. If they can, use them. If they cannot, you must call the `search_tools` function to find the right tools for the user's request. You must not use the same/similar query multiple times in the list. The list should have multiple queries only if the task has clearly different sub-tasks. If you do not find any specific relevant tools, use the pre-loaded generic tools. Only use `search_tools` if your existing capabilities cannot handle the request.
 
-3.  **Load Tools:** After looking at the output of `search_tools`, you MUST call the `load_tools` function to load only the tools you want to use. Provide the full tool ids, not just the app names. Use your judgement to eliminate irrelevant apps that came up just because of semantic similarity. However, sometimes, multiple apps might be relevant for the same task. Prefer connected apps over unconnected apps while breaking a tie. If more than one relevant app (or none of the relevant apps) are connected, you must ask the user to choose the app. In case the user asks you to use an app that is not connected, call the apps tools normally. The tool will return a link for connecting that you should pass on to the user.
+3.  **Load Tools:** After looking at the output of `search_tools`, you MUST call the `load_tools` function to load only the tools you want to use. Provide the full tool ids, not just the app names. Use your judgement to eliminate irrelevant apps that came up just because of semantic similarity. However, sometimes, multiple apps might be relevant for the same task. Prefer connected apps over unconnected apps while breaking a tie. If more than one relevant app (or none of the relevant apps) are connected, you must ask the user to choose the app. In case the user asks you to use an app that is not connected, call the apps tools normally. The tool will return a link for connecting that you should pass on to the user. Only load tools if your existing capabilities cannot handle the request.
 
 4.  **Strictly Follow the Process:** Your only job in your first turn is to analyze the user's request and answer using existing tools/knowledge or `search_tools` with a concise query describing the core task. Do not engage in conversation, or extend the conversation beyond the user's request.
 

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.14}/src/universal_mcp/agents/bigtool/tools.py

@@ -35,7 +35,8 @@ def create_meta_tools(tool_registry: ToolRegistry) -> dict[str, Any]:
                 for tool in tools_list:
                     app = tool["id"].split("__")[0]
                     if len(app_tools[app]) < 5:
-                        app_tools[app].append(f"{tool['id']}: {tool['description']}")
+                        cleaned_desc = tool['description'].split("Context:")[0].strip()
+                        app_tools[app].append(f"{tool['id']}: {cleaned_desc}")
 
             # Build result string efficiently
             result_parts = []
@@ -98,8 +99,13 @@ def create_meta_tools(tool_registry: ToolRegistry) -> dict[str, Any]:
     return {"search_tools": search_tools, "load_tools": load_tools, "web_search": web_search}
 
 
-async def get_valid_tools(tool_ids: list[str], registry: ToolRegistry) -> list[str]:
+async def get_valid_tools(tool_ids: list[str], registry: ToolRegistry) -> tuple[list[str], list[str]]:
+    """For a given list of tool_ids, validates the tools and returns a list of links for the apps that have not been logged in"""
     correct, incorrect = [], []
+    connections = await registry.list_connected_apps()
+    connected_apps = {connection["app_id"] for connection in connections}
+    unconnected = set()
+    unconnected_links = []
     app_tool_list: dict[str, set[str]] = {}
 
     # Group tool_ids by app for fewer registry calls
@@ -127,15 +133,23 @@ async def get_valid_tools(tool_ids: list[str], registry: ToolRegistry) -> list[s
             app_tool_list[app] = tools
 
     # Validate tool_ids
-    for app, tool_entries in app_to_tools.items():
+    for app, tool_entries in app_to_tools.items():        
         available = app_tool_list.get(app)
         if available is None:
             incorrect.extend(tool_id for tool_id, _ in tool_entries)
             continue
+        if app not in connected_apps and app not in unconnected:
+            unconnected.add(app)
+            text = registry.client.get_authorization_url(app)
+            start = text.find(":") + 1
+            end = text.find(".", start)
+            url = text[start:end].strip()
+            markdown_link = f"[{app}]({url})"
+            unconnected_links.append(markdown_link)
         for tool_id, tool_name in tool_entries:
             if tool_name in available:
                 correct.append(tool_id)
             else:
                 incorrect.append(tool_id)
 
-    return correct
+    return correct, unconnected_links