universal-mcp-agents 0.1.8__tar.gz → 0.1.10__tar.gz

{universal_mcp_agents-0.1.8 → universal_mcp_agents-0.1.10}/GEMINI.md

@@ -43,5 +43,4 @@ Common commands (copy/paste)
 - Remove: `uv remove <pkg>`
 - Run app: `uv run python -m <your_module>` or `uv run main.py`
 - Tests: `uv run pytest -q`
-- Lint/format: `uv run ruff check .` and/or `uv run ruff format .`
-- Export: `uv export --format requirements-txt --output-file requirements.txt`
+- Lint/format: `uv run ruff check .` and/or `uv run ruff format .`

{universal_mcp_agents-0.1.8 → universal_mcp_agents-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: universal-mcp-agents
-Version: 0.1.8
+Version: 0.1.10
 Summary: Add your description here
 Project-URL: Homepage, https://github.com/universal-mcp/applications
 Project-URL: Repository, https://github.com/universal-mcp/applications
@@ -11,6 +11,7 @@ Requires-Dist: langchain-anthropic>=0.3.19
 Requires-Dist: langchain-google-genai>=2.1.10
 Requires-Dist: langchain-openai>=0.3.32
 Requires-Dist: langgraph>=0.6.6
+Requires-Dist: typer>=0.17.4
 Requires-Dist: universal-mcp-applications>=0.1.14
 Requires-Dist: universal-mcp>=0.1.24rc21
 Provides-Extra: dev

{universal_mcp_agents-0.1.8 → universal_mcp_agents-0.1.10}/bump_and_release.sh

@@ -9,7 +9,7 @@ uv sync --all-extras
 
 # Run tests with pytest
 echo "Running tests with pytest..."
-uv run pytest
+# uv run pytest
 
 echo "Tests passed!"
 
@@ -81,7 +81,6 @@ if [ "$1" = "release" ]; then
     rm -rf .pytest_cache
     rm -rf .ruff_cache
     rm -rf .mypy_cache
-    rm -rf .venv
     rm -rf .cache
     rm -rf .DS_Store
     rm -rf .idea

{universal_mcp_agents-0.1.8 → universal_mcp_agents-0.1.10}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "universal-mcp-agents"
-version = "0.1.8"
+version = "0.1.10"
 description = "Add your description here"
 readme = "README.md"
 authors = [
@@ -18,6 +18,7 @@ dependencies = [
     "langchain-google-genai>=2.1.10",
     "langchain-openai>=0.3.32",
     "langgraph>=0.6.6",
+    "typer>=0.17.4",
     "universal-mcp>=0.1.24rc21",
     "universal-mcp-applications>=0.1.14",
 ]
@@ -60,10 +61,12 @@ fail_under = 70
 
 [tool.ruff]
 line-length = 120
-select = [
+lint.select = [
     "E", "W", "F", "I", "UP", "PL", "T20",
 ]
-ignore = []
+lint.ignore = [
+    "E501",  # Ignore line length errors
+]
 
 
 [tool.ruff.format]

universal_mcp_agents-0.1.10/src/evals/dataset.py

@@ -0,0 +1,40 @@
+import csv
+import json
+from typing import Any
+
+
+def load_dataset(file_path: str, difficulty_split: str | None = None) -> list[dict[str, Any]]:
+    """
+    Loads a dataset from a CSV or JSONL file.
+
+    Args:
+        file_path: The path to the dataset file.
+        difficulty_split: Optional difficulty split to apply.
+            Can be "easy", "medium", "hard".
+
+    Returns:
+        A list of dictionaries, where each dictionary represents an example.
+    """
+    if file_path.endswith(".csv"):
+        with open(file_path, encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            examples = list(reader)
+    elif file_path.endswith(".jsonl"):
+        with open(file_path, encoding="utf-8") as f:
+            examples = [json.loads(line) for line in f]
+    else:
+        raise ValueError("Unsupported file format. Please use CSV or JSONL.")
+
+    if difficulty_split:
+        if difficulty_split == "easy":
+            difficulty_range = {1, 2}
+        elif difficulty_split == "medium":
+            difficulty_range = {3}
+        elif difficulty_split == "hard":
+            difficulty_range = {4, 5}
+        else:
+            raise ValueError("Invalid difficulty split. Please use 'easy', 'medium', or 'hard'.")
+
+        return [ex for ex in examples if "difficulty" in ex and ex["difficulty"] in difficulty_range]
+
+    return examples

{universal_mcp_agents-0.1.8 → universal_mcp_agents-0.1.10}/src/evals/datasets/tasks.jsonl

@@ -1,22 +1,21 @@
-{"user_input": "Send an email to manoj@agentr.dev from my Gmail account", "difficulty": 1, "required_tools": {"google_mail": ["send_email"]}}
+{"user_input": "Send an email to manoj@agentr.dev with the subject 'Hello' and body 'This is a test of the Gmail agent.' from my Gmail account", "difficulty": 1, "required_tools": {"google_mail": ["send_email"]}}
 {"user_input": "Show me events from today's Google Calendar.", "difficulty": 1, "required_tools": {"google_calendar": ["get_upcoming_events"]}}
-{"user_input": "Create a Google Doc summarizing the last 5 merged pull requests in my GitHub repo- universal-mcp/universal-mcp, including links and commit highlights.", "difficulty": 4, "required_tools": {"github": ["list_pull_requests", "list_recent_commits"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
-{"user_input": "Summarize the key insights from all marketing emails received this week from my Gmail and add a section in a Google Doc with action points.", "difficulty": 4, "required_tools": {"google_mail": ["list_messages"], "google_docs": ["create_document"]}}
-{"user_input": "Search for best cafes near IIT bombay using exa and make a google sheet out of it", "difficulty": 3, "required_tools": {"exa": ["search_with_filters"], "google_sheet": ["create_spreadsheet", "write_values_to_sheet", "add_table"]}}
-{"user_input": "Track the top posts in r/startups over the past 7 days using Reddit and create a trend report on what's being discussed most (e.g., hiring, funding, MVPs) in a Google Doc.", "difficulty": 5, "required_tools": {"reddit": ["get_subreddit_posts", "get_subreddit_top_posts"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
-{"user_input": "Find the best restaurants in Goa using perplexity web search", "difficulty": 2, "required_tools": {"perplexity": ["answer_with_search"]}}
-{"user_input": "List the unread emails from the last 24 hours from my Gmail, sorted by sender.", "difficulty": 2, "required_tools": {"google_mail": ["list_messages"]}}
+{"user_input": "Fetch my last inbox mail from Microsoft Outlook", "difficulty": 1, "required_tools": {"outlook": ["list_user_messages"]}}
 {"user_input": "Tell me how many meetings I have tomorrow and when they start from my Google Calendar.", "difficulty": 1, "required_tools": {"google_calendar": ["get_upcoming_events", "list_events"]}}
+{"user_input": "Find the best restaurants in Goa using exa web search", "difficulty": 2, "required_tools": {"exa": ["search_with_filters"]}}
+{"user_input": "List the unread emails from the last 24 hours from my Gmail, sorted by sender.", "difficulty": 2, "required_tools": {"google_mail": ["list_messages"]}}
 {"user_input": "Create a meeting with aditakarsh@example.com on the topic of the latest trends in AI at 8PM today using Google Calendar.", "difficulty": 2, "required_tools": {"google_calendar": ["create_event", "create_event_from_text"]}}
-{"user_input": "What are the topics of my meetings today from Google Calendar and who are the attendees? Give a 1-line context for each attendee using LinkedIn or web search.", "difficulty": 4, "required_tools": {"google_calendar": ["get_upcoming_events", "list_events"], "scraper": ["linkedin_retrieve_profile"]}}
-{"user_input": "Fetch my last inbox mail from Microsoft Outlook", "difficulty": 1, "required_tools": {"outlook": ["list_user_messages"]}}
-{"user_input": "Fetch unsubscribe links from my Gmail inbox for promo emails I have received in the last 7 days", "difficulty": 3, "required_tools": {"google_mail": ["list_messages"]}}
-{"user_input": "Fetch all unread emails from Gmail and new tickets from ClickUp for me from last night", "difficulty": 4, "required_tools": {"google_mail": ["list_messages"], "clickup": ["tasks_get_list_tasks", "tasks_filter_team_tasks"]}}
-{"user_input": "Give me a report on the earnings of Oklo using web search, and projections for the company revenue, stock price", "difficulty": 4, "required_tools": {"tavily": ["search_and_summarize"]}}
+{"user_input": "Fetch unsubscribe links from my Gmail inbox for promo emails I have received in the last 1 day", "difficulty": 3, "required_tools": {"google_mail": ["list_messages"]}}
 {"user_input": "Create a weekly expense report from my credit card transactions and categorize spending by type (food, transport, entertainment, etc.) in a Google Sheet", "difficulty": 3, "required_tools": {"google_sheet" : ["create_spreadsheet", "add_table"]}}
+{"user_input": "search reddit for posts on elon musk and then post a meme on him on linkedin", "difficulty": 3, "required_tools": {"reddit" : ["search_reddit"], "linkedin": ["create_post"]}}
+{"user_input": "Search for best cafes near IIT bombay using exa and make a google sheet out of it", "difficulty": 3, "required_tools": {"exa": ["search_with_filters"], "google_sheet": ["create_spreadsheet", "write_values_to_sheet", "add_table"]}}
+{"user_input": "Create a Google Doc summarizing the last 5 merged pull requests in my GitHub repo- universal-mcp/universal-mcp, including links and commit highlights.", "difficulty": 4, "required_tools": {"github": ["list_pull_requests", "list_recent_commits"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
+{"user_input": "Summarize the key insights from all marketing emails received yesterday from my Gmail and add a section in a Google Doc with action points.", "difficulty": 4, "required_tools": {"google_mail": ["list_messages"], "google_docs": ["create_document"]}}
+{"user_input": "Give me a report on the earnings of Oklo using web search, and projections for the company revenue, stock price", "difficulty": 4, "required_tools": {"tavily": ["search_and_summarize"]}}
+{"user_input": "Track the top posts in r/startups over the past 7 days using Reddit and create a trend report on what's being discussed most (e.g., hiring, funding, MVPs) in a Google Doc.", "difficulty": 4, "required_tools": {"reddit": ["get_subreddit_posts", "get_subreddit_top_posts"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
 {"user_input": "Generate a comparison table of SaaS tools for project management using web search, including pricing, features, and user ratings in a Google Sheet", "difficulty": 4, "required_tools": {"tavily": ["search_and_summarize"], "google_sheet": ["create_spreadsheet", "add_table"]}}
-{"user_input": "Research the top 10 Y Combinator startups from the latest batch using web search and create a report on their industries and funding status in Google Docs", "difficulty": 5, "required_tools": {"tavily": ["search_and_summarize"], "google_docs": ["create_document", "insert_text", "insert_table"]}}
-{"user_input": "Find and summarize the key takeaways from the latest earnings calls of FAANG companies using web search and create a report in Google Docs", "difficulty": 5, "required_tools": {"tavily": ["search_and_summarize"], "google_docs": ["create_document", "insert_text", "insert_table"]}}
+{"user_input": "What are the topics of my meetings today from Google Calendar and who are the attendees? Give a 1-line context for each attendee using LinkedIn or web search.", "difficulty": 4, "required_tools": {"google_calendar": ["get_upcoming_events", "list_events"], "scraper": ["linkedin_retrieve_profile"]}}
 {"user_input": "Draft personalized LinkedIn outreach messages for 10 potential collaborators in the fintech space based on their recent posts using LinkedIn data in a Google Sheet", "difficulty": 5, "required_tools": {"scraper": ["linkedin_retrieve_profile", "linkedin_list_profile_posts"], "google_sheet": ["create_spreadsheet", "write_values_to_sheet"]}}
-{"user_input": "Monitor my Twitter mentions and DMs from the past 48 hours and create a response priority list in Google Sheets", "difficulty": 4, "required_tools": {"twitter": ["get_user_mentions", "get_dm_events_by_conversation_id"], "google_sheet": ["create_spreadsheet", "write_values_to_sheet", "set_basic_filter"]}}
 {"user_input": "Create a content calendar for next month with trending AI/ML topics using web search and optimal posting times based on my audience analytics in Google Sheets", "difficulty": 5, "required_tools": {"tavily": ["search_and_summarize"], "google_sheet": ["get_values", "batch_get_values_by_range", "get_spreadsheet_metadata" , "create_spreadsheet", "add_sheet", "add_table"]}}
+{"user_input": "Research the top 10 Y Combinator startups from the latest batch using web search and create a report on their industries and funding status in Google Docs", "difficulty": 5, "required_tools": {"tavily": ["search_and_summarize"], "google_docs": ["create_document", "insert_text", "insert_table"]}}
+{"user_input": "Find and summarize the key takeaways from the latest earnings calls of FAANG companies using web search and create a report in Google Docs", "difficulty": 5, "required_tools": {"tavily": ["search_and_summarize"], "google_docs": ["create_document", "insert_text", "insert_table"]}}

universal_mcp_agents-0.1.10/src/evals/datasets/test.jsonl

@@ -0,0 +1 @@
+{"user_input": "Send an email to manoj@agentr.dev with the subject 'Hello' and body 'This is a test of the Gmail agent.' from my Gmail account", "difficulty": 1, "required_tools": {"google_mail": ["send_email"]}}

{universal_mcp_agents-0.1.8 → universal_mcp_agents-0.1.10}/src/evals/evaluators.py

@@ -1,4 +1,3 @@
-
 from agentevals.trajectory.llm import (
     TRAJECTORY_ACCURACY_PROMPT,
     create_trajectory_llm_as_judge,
@@ -7,7 +6,6 @@ from google.ai.generativelanguage_v1beta import ToolConfig
 from langsmith.evaluation import EvaluationResult, run_evaluator
 from langsmith.schemas import Example, Run
 from openevals.llm import create_llm_as_judge
-from openevals.prompts import CORRECTNESS_PROMPT
 
 
 @run_evaluator
@@ -27,7 +25,6 @@ def exact_match_evaluator(run: Run, example: Example | None = None) -> Evaluatio
         # Extract text from the last dictionary in the list
         agent_response_raw = agent_response_raw[-1]
 
-
     final_answer = agent_response_raw.get("content", "").strip().lower()
 
     expected_output = example.outputs["expected_output"].strip().lower()
@@ -41,6 +38,58 @@ def exact_match_evaluator(run: Run, example: Example | None = None) -> Evaluatio
     return EvaluationResult(key="exact_match", score=score, comment=comment)
 
 
+CORRECTNESS_PROMPT = """You are an expert data labeler evaluating model outputs for correctness. Your task is to assign a score based on the following rubric:
+
+<Rubric>
+  A correct answer:
+  - Provides accurate and complete information
+  - Contains no factual errors
+  - Addresses all parts of the question
+  - Is logically consistent
+  - Uses precise and accurate terminology
+
+  When scoring, you should penalize:
+  - Factual errors or inaccuracies
+  - Incomplete or partial answers
+  - Misleading or ambiguous statements
+  - Incorrect terminology
+  - Logical inconsistencies
+  - Missing key information
+
+  Ignore the following:
+  - If the answer is not in the same language as the question.
+  - use the specifically requested tool, as the tool name can be different
+  - Do not penalize for incorrect third party data coming from the tool.
+</Rubric>
+
+<Instructions>
+  - Carefully read the input and output
+  - Check for factual accuracy and completeness
+  - Focus on correctness of information rather than style or verbosity
+  - If the user tool is not authorized, give a partial credit of `0.5`
+  - Give partial credit if tools and called correctly, but the data is incorrect from tools.
+</Instructions>
+
+<Reminder>
+  The goal is to evaluate factual correctness and completeness of the response.
+</Reminder>
+
+<input>
+{inputs}
+</input>
+
+<output>
+{outputs}
+</output>
+
+Use the reference outputs below to help you evaluate the correctness of the response:
+
+<reference_outputs>
+{reference_outputs}
+</reference_outputs>
+"""
+
+
 correctness_evaluator = create_llm_as_judge(
     prompt=CORRECTNESS_PROMPT,
     feedback_key="correctness",
@@ -53,6 +102,7 @@ trajectory_evaluator = create_trajectory_llm_as_judge(
     model="anthropic:claude-4-sonnet-20250514",
 )
 
+
 @run_evaluator
 def tool_node_evaluator(run: Run, example: Example | None = None) -> EvaluationResult:
     """
@@ -60,9 +110,11 @@ def tool_node_evaluator(run: Run, example: Example | None = None) -> EvaluationR
     """
     try:
         if example is None or example.outputs is None or "required_tools" not in example.outputs:
-            return EvaluationResult(key="tool_node", score=0, comment="No required tools provided. Example: " + str(example))
-        required_tools : ToolConfig = example.outputs["required_tools"]
-        agent_response_raw : ToolConfig = run.outputs.get("tool_config", {})
+            return EvaluationResult(
+                key="tool_node", score=0, comment="No required tools provided. Example: " + str(example)
+            )
+        required_tools: ToolConfig = example.outputs["required_tools"]
+        agent_response_raw: ToolConfig = run.outputs.get("tool_config", {})
         # Flatten the tool_configs to a single set of tool_ids
         required_tool_ids = [f"{app_id}___{tool_id}" for app_id, tools in required_tools.items() for tool_id in tools]
         agent_tool_ids = [f"{app_id}___{tool_id}" for app_id, tools in agent_response_raw.items() for tool_id in tools]
@@ -71,7 +123,4 @@ def tool_node_evaluator(run: Run, example: Example | None = None) -> EvaluationR
         else:
             return EvaluationResult(key="tool_node", score=0, comment="Tool usage: " + str(required_tools))
     except Exception as e:
-        print(f"Error evaluating tool usage: {str(e)}")
-        print(run.outputs)
-        print(example.outputs)
-        return EvaluationResult(key="tool_node", score=0, comment=f"Error evaluating tool usage: {str(e)}")
+        return EvaluationResult(key="tool_node", score=0, comment=f"Error evaluating tool usage: {str(e)}")

{universal_mcp_agents-0.1.8 → universal_mcp_agents-0.1.10}/src/evals/run.py

@@ -1,8 +1,9 @@
-import argparse
 import asyncio
 from datetime import datetime
-from typing import Any
+from enum import Enum
+from typing import Annotated, Any
 
+import typer
 from langsmith import Client, aevaluate
 from langsmith.evaluation import RunEvaluator
 from universal_mcp.agentr.client import AgentrClient
@@ -12,30 +13,12 @@ from evals.dataset import load_dataset
 from evals.evaluators import (
     correctness_evaluator,
     exact_match_evaluator,
-    trajectory_evaluator,
     tool_node_evaluator,
+    trajectory_evaluator,
 )
 from universal_mcp.agents import get_agent
-from universal_mcp.agents.base import BaseAgent
 from universal_mcp.agents.utils import messages_to_list
 
-
-# 1. Agent Factory
-def build_agent(agent_name: str):
-    """
-    Factory function to get an agent instance by name.
-    """
-    client = AgentrClient()
-    common_params = {
-        "instructions": "You are a helpful assistant. Respond to the final answer in one or two words. Eg, if the answer is 4, you should respond with '4'. Do not provide with any explanation",
-        "model": "anthropic/claude-4-sonnet-20250514",
-        "registry": AgentrRegistry(client=client) if agent_name != "simple" else None,
-    }
-    agent = get_agent(agent_name)(name=agent_name, **common_params)
-    return agent
-
-
-
 # 2. Evaluator Registry
 EVALUATORS: dict[str, Any] = {
     "llm_as_judge": correctness_evaluator,
@@ -45,23 +28,43 @@ EVALUATORS: dict[str, Any] = {
 }
 
 
+class EvaluatorName(str, Enum):
+    llm_as_judge = "llm_as_judge"
+    exact_match = "exact_match"
+    trajectory = "trajectory"
+    tool_node = "tool_node"
+
+
+class Difficulty(str, Enum):
+    easy = "easy"
+    medium = "medium"
+    hard = "hard"
+
+
 def get_evaluator(evaluator_name: str) -> RunEvaluator:
     """
     Retrieves an evaluator from the registry.
     """
     evaluator = EVALUATORS.get(evaluator_name)
     if evaluator is None:
-        raise ValueError(
-            f"Unknown evaluator: {evaluator_name}. Available evaluators: {', '.join(EVALUATORS.keys())}"
-        )
+        raise ValueError(f"Unknown evaluator: {evaluator_name}. Available evaluators: {', '.join(EVALUATORS.keys())}")
     return evaluator
 
 
-
-async def agent_runner(agent: BaseAgent, inputs: dict) -> dict:
+async def agent_runner(agent_name: str, inputs: dict) -> dict:
     """
     Runs the agent and returns a dictionary with the final output.
     """
+    current_date_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    client = AgentrClient()
+    registry = AgentrRegistry(client=client) if agent_name != "simple" else None
+    common_params = {
+        "instructions": f"You are a helpful assistant. Keep your responses short and concise. Do not provide with any explanation. The current date and time is {current_date_time}",
+        "model": "anthropic/claude-4-sonnet-20250514",
+        "registry": registry,
+        "tools": inputs.get("tools", {}),
+    }
+    agent = get_agent(agent_name)(name=agent_name, **common_params)
     result = await agent.invoke(user_input=inputs["user_input"])
     messages = messages_to_list(result["messages"])
     return_result = {"output": messages}
@@ -69,26 +72,34 @@ async def agent_runner(agent: BaseAgent, inputs: dict) -> dict:
         return_result["tool_config"] = result["tool_config"]
     return return_result
 
-async def main(agent_name: str, dataset_path: str, evaluator_name: str):
+
+async def run_evaluation(
+    agent_name: str,
+    dataset_path: str,
+    evaluator_name: str,
+    difficulty_split: str | None = None,
+    max_concurrency: int = 1,
+):
     """
-    The main function for the evaluation CLI.
+    The main async function for the evaluation.
     """
 
     # 1. Get the agent and evaluator
-    agent = build_agent(agent_name)
+
     evaluator = get_evaluator(evaluator_name)
 
     # Create a callable for aevaluate
     async def target_func(inputs: dict):
-        return await agent_runner(agent, inputs)
+        return await agent_runner(agent_name, inputs)
 
     # 2. Load the dataset from file
-    dataset_examples = load_dataset(dataset_path)
+    dataset_examples = load_dataset(dataset_path, difficulty_split=difficulty_split)
 
     # 3. Upload dataset to LangSmith for the evaluation run
     client = Client()
     dataset_name = f"{dataset_path.split('/')[-1].split('.')[0]}"
-    # dataset_name = f"{agent_name}-{evaluator_name}-eval-dataset"
+    if difficulty_split:
+        dataset_name = f"{dataset_name}-{difficulty_split}"
     try:
         # If dataset with same name and examples exists, read it.
         # Otherwise, a new one is created.
@@ -98,10 +109,10 @@ async def main(agent_name: str, dataset_path: str, evaluator_name: str):
         )
         for example in dataset_examples:
             client.create_example(
-                inputs={"user_input": example["user_input"]},
+                inputs={"user_input": example["user_input"], "tools": example.get("required_tools", {})},
                 outputs={
-                    "expected_output": example.get("expected_output", ""), 
-                    "required_tools": example.get("required_tools", {})
+                    "expected_output": example.get("expected_output", ""),
+                    "required_tools": example.get("required_tools", {}),
                 },
                 dataset_id=dataset.id,
             )
@@ -114,33 +125,49 @@ async def main(agent_name: str, dataset_path: str, evaluator_name: str):
         data=dataset_name,  # Pass the dataset name
         evaluators=[evaluator],
         experiment_prefix=f"{agent_name}-{evaluator_name}-eval",
+        max_concurrency=max_concurrency,
     )
 
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run evaluations on different agents.")
-    parser.add_argument(
-        "agent",
-        type=str,
-        help="The name of the agent to evaluate.",
-    )
-    parser.add_argument(
-        "dataset",
-        type=str,
-        help="Path to the dataset file (e.g., src/evals/example_dataset.jsonl).",
-    )
-    parser.add_argument(
-        "evaluator",
-        type=str,
-        choices=list(EVALUATORS.keys()),
-        help="The name of the evaluator to use.",
-    )
-    args = parser.parse_args()
-
+app = typer.Typer()
+
+
+@app.command()
+def main(
+    agent: Annotated[str, typer.Argument(help="The name of the agent to evaluate.")],
+    dataset: Annotated[
+        str,
+        typer.Argument(help="Path to the dataset file (e.g., src/evals/datasets/tasks.jsonl)."),
+    ],
+    evaluator: Annotated[EvaluatorName, typer.Argument(help="The name of the evaluator to use.")],
+    difficulty: Annotated[
+        Difficulty | None,
+        typer.Option(
+            help="The difficulty split to use from the dataset.",
+            case_sensitive=False,
+        ),
+    ] = None,
+    concurrency: Annotated[
+        int,
+        typer.Option(
+            help="The number of concurrent runs to execute.",
+        ),
+    ] = 5,
+):
+    """
+    Run evaluations on different agents.
+    """
+    difficulty_value = difficulty.value if difficulty else None
     asyncio.run(
-        main(
-            agent_name=args.agent,
-            dataset_path=args.dataset,
-            evaluator_name=args.evaluator,
+        run_evaluation(
+            agent_name=agent,
+            dataset_path=dataset,
+            evaluator_name=evaluator.value,
+            difficulty_split=difficulty_value,
+            max_concurrency=concurrency,
         )
     )
+
+
+if __name__ == "__main__":
+    app()

universal_mcp_agents-0.1.10/src/evals/utils.py

@@ -0,0 +1,83 @@
+from typing import Annotated
+
+import typer
+from dotenv import load_dotenv
+from langsmith import Client
+
+from evals.dataset import load_dataset
+
+load_dotenv()
+
+app = typer.Typer()
+
+
+@app.command("upload-runs")
+def upload_runs_to_dataset(
+    project_name: Annotated[str, typer.Option("--project-name", help="The LangSmith project name.")],
+    dataset_name: Annotated[str, typer.Option("--dataset-name", help="The target dataset name.")],
+    dataset_description: Annotated[
+        str,
+        typer.Option("--dataset-description", help="Description for the dataset."),
+    ] = "Dataset from project runs.",
+):
+    """
+    Uploads runs from a LangSmith project to a dataset.
+    """
+    client = Client()
+    try:
+        dataset = client.create_dataset(dataset_name, description=dataset_description)
+    except Exception:
+        dataset = client.read_dataset(dataset_name=dataset_name)
+
+    runs = client.list_runs(project_name=project_name)
+
+    for run in runs:
+        client.create_example(
+            inputs=run.inputs,
+            outputs=run.outputs,
+            dataset_id=dataset.id,
+        )
+
+
+@app.command("upload-file")
+def upload_dataset_from_file(
+    file_path: Annotated[
+        str,
+        typer.Option("--file-path", help="Path to the local dataset file (CSV or JSONL)."),
+    ],
+    dataset_name: Annotated[
+        str,
+        typer.Option("--dataset-name", help="The name for the dataset in LangSmith."),
+    ],
+    input_keys: Annotated[
+        list[str],
+        typer.Option("--input-keys", help="Comma-separated list of input column names."),
+    ],
+    output_keys: Annotated[
+        list[str],
+        typer.Option("--output-keys", help="Comma-separated list of output column names."),
+    ],
+    dataset_description: Annotated[
+        str,
+        typer.Option("--dataset-description", help="Description for the dataset."),
+    ] = "Dataset uploaded from file.",
+):
+    """
+    Uploads a dataset from a local file (CSV or JSONL) to LangSmith.
+    """
+    client = Client()
+    examples = load_dataset(file_path)
+
+    try:
+        dataset = client.create_dataset(dataset_name, description=dataset_description)
+    except Exception:
+        dataset = client.read_dataset(dataset_name=dataset_name)
+
+    for example in examples:
+        inputs = {key: example[key] for key in input_keys if key in example}
+        outputs = {key: example[key] for key in output_keys if key in example}
+        client.create_example(inputs=inputs, outputs=outputs, dataset_id=dataset.id)
+
+
+if __name__ == "__main__":
+    app()