universal-mcp-agents 0.1.13__tar.gz → 0.1.15__tar.gz

universal_mcp_agents-0.1.15/.github/workflows/evals.yml

@@ -0,0 +1,61 @@
+name: evals
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  eval:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { agent: bigtool, dataset: src/evals/datasets/tasks.jsonl,  evaluator: llm_as_judge, difficulty: easy }
+          - { agent: codeact-repl, dataset: src/evals/datasets/codeact.jsonl, evaluator: codeact, difficulty: none }
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install dependencies
+        run: |
+          uv sync --all-extras
+
+      - name: Run eval
+        env:
+          PYTHONUNBUFFERED: '1'
+        run: |
+          mkdir -p dist/evals/${{ matrix.agent }}/${{ matrix.evaluator }}
+          if [ "${{ matrix.difficulty }}" = "none" ]; then
+            uv run python -u src/evals/run.py \
+              ${{ matrix.agent }} \
+              ${{ matrix.dataset }} \
+              ${{ matrix.evaluator }} \
+              2>&1 | tee dist/evals/${{ matrix.agent }}/${{ matrix.evaluator }}/run-${{ matrix.agent }}-${{ matrix.evaluator }}-$(basename ${{ matrix.dataset }} .jsonl)-none.log
+          else
+            uv run python -u src/evals/run.py \
+              ${{ matrix.agent }} \
+              ${{ matrix.dataset }} \
+              ${{ matrix.evaluator }} \
+              --difficulty ${{ matrix.difficulty }} \
+              2>&1 | tee dist/evals/${{ matrix.agent }}/${{ matrix.evaluator }}/run-${{ matrix.agent }}-${{ matrix.evaluator }}-$(basename ${{ matrix.dataset }} .jsonl)-${{ matrix.difficulty }}.log
+          fi
+
+      - name: Upload eval artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-artifacts-${{ matrix.agent }}-${{ matrix.evaluator }}
+          path: dist/evals/${{ matrix.agent }}/${{ matrix.evaluator }}
+          if-no-files-found: warn
+
+

universal_mcp_agents-0.1.15/.github/workflows/lint.yml

@@ -0,0 +1,31 @@
+name: lint
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install dependencies
+        run: |
+          uv sync --all-extras
+
+      - name: Run ruff
+        run: |
+          uv run ruff check .
+          uv run ruff format --check .

universal_mcp_agents-0.1.15/.github/workflows/release-please.yml

@@ -0,0 +1,67 @@
+name: release-please
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: googleapis/release-please-action@v4
+        id: release
+        with:
+          release-type: python
+          package-name: agents
+          default-branch: main
+
+      - name: Checkout repository
+        if: ${{ steps.release.outputs.release_created }}
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 0
+
+      - name: Add Authors and Update Release
+        if: ${{ steps.release.outputs.release_created }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_TAG: ${{ steps.release.outputs.tag_name }}
+        run: |
+          set -e
+
+          TEMP_CHANGELOG="CHANGELOG.md.new"
+          touch "$TEMP_CHANGELOG"
+
+          if [ -f "CHANGELOG.md" ]; then
+            while IFS= read -r line; do
+              if [[ "$line" =~ \*\ (.*)\ \(\[([a-f0-9]{7,40})\]\(.* ]]; then
+                commit_hash="${BASH_REMATCH[2]}"
+                github_user=$(gh api "repos/${{ github.repository }}/commits/${commit_hash}"  | jq -r '.author.login // "unknown"')
+                echo "${line} by @${github_user}" >> "$TEMP_CHANGELOG"
+              else
+                echo "$line" >> "$TEMP_CHANGELOG"
+              fi
+            done < "CHANGELOG.md"
+          fi
+
+          mv "$TEMP_CHANGELOG" "CHANGELOG.md"
+
+          RELEASE_BODY=$(awk "/^## \\[?${RELEASE_TAG#v}/{flag=1;next} /^## / && flag{exit} flag" CHANGELOG.md)
+          if [ -z "$RELEASE_BODY" ]; then
+            RELEASE_BODY="Release $RELEASE_TAG"
+          fi
+          gh release edit "$RELEASE_TAG" --notes "$RELEASE_BODY"
+
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add CHANGELOG.md || true
+          git commit -m "update CHANGELOG.md with author info [skip ci] [skip release]" || true
+          git push || true
+
+

universal_mcp_agents-0.1.15/.github/workflows/tests.yml

@@ -0,0 +1,39 @@
+name: tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install dependencies
+        run: |
+          uv sync --all-extras
+
+      - name: Run tests
+        run: |
+          uv run pytest -s -q --maxfail=1 --disable-warnings --junitxml=junit.xml
+        env:
+          PYTHONUNBUFFERED: "1"
+
+      - name: Store artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-artifacts
+          path: junit.xml
+          if-no-files-found: ignore

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.15}/.pre-commit-config.yaml

@@ -1,20 +1,16 @@
 fail_fast: false
 
 repos:
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.1.0
-    hooks:
-      - id: prettier
-        types_or: [yaml, json5]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.13
+    rev: v0.13.3
     hooks:
       # Run the linter.
       - id: ruff-check
-        args: [--fix]
+        args: ["--fix", "src", "--config", "pyproject.toml"]
       # Run the formatter.
       - id: ruff-format
+        args: ["src", "--config", "pyproject.toml"]
 
   # - repo: https://github.com/pre-commit/mirrors-mypy
   #   rev: v1.8.0

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: universal-mcp-agents
-Version: 0.1.13
+Version: 0.1.15
 Summary: Add your description here
 Project-URL: Homepage, https://github.com/universal-mcp/applications
 Project-URL: Repository, https://github.com/universal-mcp/applications
@@ -12,8 +12,8 @@ Requires-Dist: langchain-google-genai>=2.1.10
 Requires-Dist: langchain-openai>=0.3.32
 Requires-Dist: langgraph>=0.6.6
 Requires-Dist: typer>=0.17.4
-Requires-Dist: universal-mcp-applications>=0.1.14
-Requires-Dist: universal-mcp>=0.1.24rc21
+Requires-Dist: universal-mcp-applications>=0.1.24
+Requires-Dist: universal-mcp>=0.1.24rc25
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
 Requires-Dist: ruff; extra == 'dev'

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.15}/bump_and_release.sh

@@ -9,7 +9,7 @@ uv sync --all-extras
 
 # Run tests with pytest
 echo "Running tests with pytest..."
-# uv run pytest
+uv run pytest
 
 echo "Tests passed!"
 

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.15}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "universal-mcp-agents"
-version = "0.1.13"
+version = "0.1.15"
 description = "Add your description here"
 readme = "README.md"
 authors = [
@@ -19,8 +19,8 @@ dependencies = [
     "langchain-openai>=0.3.32",
     "langgraph>=0.6.6",
     "typer>=0.17.4",
-    "universal-mcp>=0.1.24rc21",
-    "universal-mcp-applications>=0.1.14",
+    "universal-mcp>=0.1.24rc25",
+    "universal-mcp-applications>=0.1.24",
 ]
 
 [project.license]
@@ -51,7 +51,7 @@ packages = [
 
 [tool.coverage.run]
 source = [
-    "src",   
+    "src",
 ]
 branch = true
 
@@ -68,6 +68,12 @@ lint.ignore = [
     "E501",  # Ignore line length errors
 ]
 
+[tool.ruff.lint.pylint]
+max-args = 10
+max-statements = 85
+max-returns = 10
+max-branches = 37
+
 
 [tool.ruff.format]
 quote-style = "double"
@@ -83,4 +89,3 @@ asyncio_default_fixture_loop_scope = "module"
 dev = [
     "ruff>=0.13.0",
 ]
-

universal_mcp_agents-0.1.15/src/evals/datasets/tasks.jsonl

@@ -0,0 +1,32 @@
+{"user_input": "Send an email to manoj@agentr.dev with the subject 'Hello' and body 'This is a test of the Gmail agent.' from my Gmail account", "difficulty": 1, "required_tools": {"google_mail": ["send_email"]}}
+{"user_input": "Show me events from today's Google Calendar.", "difficulty": 1, "required_tools": {"google_calendar": ["get_upcoming_events"]}}
+{"user_input": "Fetch my last inbox mail from Microsoft Outlook", "difficulty": 1, "required_tools": {"outlook": ["list_user_messages"]}}
+{"user_input": "Tell me how many meetings I have tomorrow and when they start from my Google Calendar.", "difficulty": 1, "required_tools": {"google_calendar": ["get_upcoming_events", "list_events"]}}
+{"user_input": "Find the best restaurants in Goa using exa web search", "difficulty": 2, "required_tools": {"exa": ["search_with_filters"]}}
+{"user_input": "List the unread emails from the last 24 hours from my Gmail, sorted by sender.", "difficulty": 2, "required_tools": {"google_mail": ["list_messages"]}}
+{"user_input": "Create a meeting with aditakarsh@example.com on the topic of the latest trends in AI at 8PM today using Google Calendar.", "difficulty": 2, "required_tools": {"google_calendar": ["create_event", "create_event_from_text"]}}
+{"user_input": "Fetch unsubscribe links from my Gmail inbox for promo emails I have received in the last 1 day", "difficulty": 3, "required_tools": {"google_mail": ["list_messages"]}}
+{"user_input": "Create a weekly expense report from my credit card transactions and categorize spending by type (food, transport, entertainment, etc.) in a Google Sheet", "difficulty": 3, "required_tools": {"google_sheet" : ["create_spreadsheet", "add_table"]}}
+{"user_input": "search reddit for posts on elon musk and then post a meme on him on linkedin", "difficulty": 3, "required_tools": {"reddit" : ["search_reddit"], "linkedin": ["create_post"]}}
+{"user_input": "Search for best cafes near IIT bombay using exa and make a google sheet out of it", "difficulty": 3, "required_tools": {"exa": ["search_with_filters"], "google_sheet": ["create_spreadsheet", "write_values_to_sheet", "add_table"]}}
+{"user_input": "Create a Google Doc summarizing the last 5 merged pull requests in my GitHub repo- universal-mcp/universal-mcp, including links and commit highlights.", "difficulty": 4, "required_tools": {"github": ["list_pull_requests", "list_recent_commits"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
+{"user_input": "Summarize the key insights from all marketing emails received yesterday from my Gmail and add a section in a Google Doc with action points.", "difficulty": 4, "required_tools": {"google_mail": ["list_messages"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
+{"user_input": "Give me a report on the earnings of Oklo using web search, and projections for the company revenue, stock price", "difficulty": 4, "required_tools": {"tavily": ["search_and_summarize"]}}
+{"user_input": "Track the top posts in r/startups over the past 7 days using Reddit and create a trend report on what's being discussed most (e.g., hiring, funding, MVPs) in a Google Doc.", "difficulty": 4, "required_tools": {"reddit": ["get_subreddit_posts", "get_subreddit_top_posts"], "google_docs": ["create_document", "insert_text", "apply_text_style"]}}
+{"user_input": "Generate a comparison table of SaaS tools for project management using web search, including pricing, features, and user ratings in a Google Sheet", "difficulty": 4, "required_tools": {"tavily": ["search_and_summarize"], "google_sheet": ["create_spreadsheet", "add_table"]}}
+{"user_input": "What are the topics of my meetings today from Google Calendar and who are the attendees? Give a 1-line context for each attendee using LinkedIn or web search.", "difficulty": 4, "required_tools": {"google_calendar": ["get_upcoming_events", "list_events"], "scraper": ["linkedin_retrieve_profile"]}}
+{"user_input": "Draft personalized LinkedIn outreach messages for 10 potential collaborators in the fintech space based on their recent posts using LinkedIn data in a Google Sheet", "difficulty": 5, "required_tools": {"scraper": ["linkedin_retrieve_profile", "linkedin_list_profile_posts"], "google_sheet": ["create_spreadsheet", "write_values_to_sheet"]}}
+{"user_input": "Create a content calendar for next month with trending AI/ML topics using web search and optimal posting times based on my audience analytics in Google Sheets", "difficulty": 5, "required_tools": {"tavily": ["search_and_summarize"], "google_sheet": ["get_values", "batch_get_values_by_range", "get_spreadsheet_metadata" , "create_spreadsheet", "add_sheet", "add_table"]}}
+{"user_input": "Research the top 10 Y Combinator startups from the latest batch using web search and create a report on their industries and funding status in Google Docs", "difficulty": 5, "required_tools": {"tavily": ["search_and_summarize"], "google_docs": ["create_document", "insert_text", "insert_table"]}}
+{"user_input": "Find and summarize the key takeaways from the latest earnings calls of FAANG companies using web search and create a report in Google Docs", "difficulty": 5, "required_tools": {"tavily": ["search_and_summarize"], "google_docs": ["create_document", "insert_text", "insert_table"]}}
+{"user_input": "Find and extract unsubscribe links from all emails in my inbox from the last 7 days. List all unsubscribe links found with the email subject and sender.", "difficulty": 3, "required_tools": {"google_mail": ["list_messages", "get_message_details"]}}
+{"user_input": "Process rows 2-5 from the Google Sheet (ID: 1nnnCp3_IWcdHv4UVgXtwYF5wedxbqF4RIeyjN6mCKD8). For each unprocessed row, extract Reddit post links, fetch post details and comments, analyze content relevance to AgentR/Wingmen products, classify into tiers 1-4, generate appropriate response drafts, and update the sheet with all findings.", "difficulty": 5, "required_tools": {"google_sheet": ["add_table", "append_values", "update_values", "format_cells", "get_spreadsheet_metadata", "batch_get_values_by_range"], "reddit": ["get_post_comments_details"], "google_mail": ["list_messages"]}}
+{"user_input": "Fetch all open issues from the GitHub repository \"microsoft/vscode\" and add them to a new Google Sheet. Then create corresponding tasks in ClickUp for each issue with descriptions, tags, and \"In Progress\" status. Delete processed rows from the sheet after creating ClickUp tasks.", "difficulty": 5, "required_tools": {"google_sheet": ["get_values", "create_spreadsheet", "write_values_to_sheet", "delete_dimensions", "append_values", "update_values"], "clickup": ["tasks_create_new_task", "spaces_get_details", "lists_get_list_details", "tasks_get_list_tasks"], "github": ["search_issues", "update_issue"]}}
+{"user_input": "Goal: Process unprocessed rows in a fixed Google Sheet, scrape Reddit for context, filter posts, and generate short, natural comments linking to AgentR/Wingmen when relevant. Workflow: 1) Sheet & Row Selection: Fixed Sheet ID 1nnnCp3_IWcdHv4UVgXtwYF5wedxbqF4RIeyjN6mCKD8, tab Posts. Process rows 2-5 (first 4 unprocessed rows) immediately without asking for user input. Only process rows with empty Match Type (Col I) and no Tier 1-4 assigned. 2) Reddit Context Fetch: Extract Post Link & ID. Use reddit to fetch post upvotes + top comments (max 5). Ensure post/comment is active, visible, and unlocked. 3) Filtration & Fit: Classify content (developer, consumer, anecdotal). Apply GTM Filtration to skip irrelevant, negative, political, or low-quality posts. Identify direct or adjacent fit to AgentR (Universal MCP Server) or Wingmen. Decide platform + account type: Direct fit/competitor mention → Technical Q = Team account, Non-technical = Burner account. Adjacent fit → Official account. Decide reply target (original comment/post or parent post). 4) Comment Generation: For Tier 1-3, craft a 2-3 line, context-aware, conversational reply. Mention AgentR/Wingmen organically, avoid sales tone or forced CTAs. Use light imperfections for human tone. Skip negative sentiment entirely. One comment per post. 5) Populate Output: Fill Upvote Count, Match Type, Account Type, Response Draft, Respond on. Return updated Google Sheet link. Tier Definitions: Tier 1 = Deep MCP, AI agent, tool integrations, or architecture discussions where infra is highly relevant. Tier 2 = Specific workflows, automation tooling, or productivity systems where Wingmen or MCP Server could be useful. Tier 3 = Broader ecosystem (LangChain/CrewAI/agent tooling) where a soft recommendation adds value. Tier 4 = Unclear, generic, sarcastic, hostile, or irrelevant mentions — skip. Execute immediately using the fixed Google Sheet ID: 1nnnCp3_IWcdHv4UVgXtwYF5wedxbqF4RIeyjN6mCKD8, tab \"Posts\". Process rows(first 4 unprocessed rows) without asking for user input. Only process rows where Match Type (Column I) is empty. For each row, extract the Post Link, fetch Reddit data, apply GTM filtration, generate appropriate responses, and update the sheet. Return the updated Google Sheet link when complete.", "difficulty": 5, "required_tools": {"reddit": ["get_post_comments_details"], "google_sheet": ["update_values", "get_values", "get_spreadsheet_metadata", "batch_get_values_by_range"]}}
+{"user_input": "Generate a financial flash report for Apple Inc. Research their latest earnings data including revenue, net income, EPS, and year-over-year changes. Create a formatted report with highlights, upcoming events, and summary. Present the report in chat and email it to adit@agentr.dev.", "difficulty": 4, "required_tools": {"exa": ["answer"], "google_mail": ["send_email"]}}
+{"user_input": "Objective: Find businesses from Google Maps for a given category & location, store them in a Google Sheet, then process unprocessed leads to scrape emails and sync with HubSpot CRM. Stage 1 - Lead Discovery Get coordinates of Area + City. Search on Google Maps with category & coordinates. Extract: Name, Google Maps URL, Address, Phone, Website; leave Email & CRM Status blank. Sheet: Name: {Area}, {City} Leads - {Category} - {dd-mmm} If exists → append non-duplicate rows; else create in folder \"Leads from Google Maps\" (ID: 142QBejJX0jAqzDz_NHdwVTkcmagoog__). Add headers: Name | Google Maps URL | Address | Phone | Website | Email | CRM Status. Populate with businesses found. Edge Cases: No results → return message, skip sheet creation. Missing data → leave blank. Stage 2 - Lead Processing & CRM Sync Locate sheet in Google Drive, ensure headers match. Parse category from sheet name. Identify unprocessed rows (CRM Status blank) — by default process the first, or a specified row/range/count. Scrape Website for Email: If website exists → scrape homepage/contact page; fallback to firecrawl_scrape_url. Save found email in sheet. HubSpot Handling: Search contact by email/website/phone. If not found → create with available details, Lead Status = New, add note {Area, City} — {Category} — {Google Maps URL}. If exists → append note; keep other fields unchanged. Save HubSpot Contact URL/ID in sheet. Update CRM Status: Lead Created, Lead Creation Failed, Website not found, Email not found, etc. Edge Cases: No Website → create with phone; mark Website not found. No Email → create; mark Email not found. Email already in sheet → skip row. Execute immediately for \"Cafes\" near \"IIT Bombay\" in \"Mumbai\" without asking for confirmation.", "difficulty": 5, "required_tools": {"serpapi": ["google_maps_search"], "firecrawl": ["scrape_url"], "google_drive": ["get_file_details", "create_folder", "find_folder_id_by_name", "search_files"], "google_sheet": ["update_values", "get_values", "get_spreadsheet_metadata", "batch_get_values_by_range", "create_spreadsheet", "clear_values"], "hubspot": ["search_contacts_post", "batch_read_contacts_post", "get_contacts", "get_contact_by_id", "update_contact_by_id", "batch_update_contacts", "create_contacts_batch", "create_contact"]}}
+{"user_input": "Process emails from the last 24 hours. Fetch primary inbox emails excluding replied threads, classify with LLM as Reply Required, No Reply Needed, or Ambiguous. For Reply Required/Ambiguous, draft human, on-brand replies for user review. Follow greeting, acknowledge, address concern, invite further questions, and friendly sign-off. Provide end summary of drafts, skipped, and ambiguous emails. Execute immediately without asking for confirmation. Do not send any emails. Just provide me a report.", "difficulty": 4, "required_tools": {"google_mail": ["list_messages", "get_message_details"]}}
+{"user_input": "Analyze a contract from my google drive from the perspective of the Service Provider. Use the search to find it, do not ask me any questions, and assume details that I have not provided. Identify potentially unfavorable clauses such as vague terms, one-sided obligations, IP transfer issues, indemnity clauses, termination conditions, and payment problems. Provide a structured analysis with clause numbers, full text, and explanations of concerns.", "difficulty": 4, "required_tools": {"google_drive": ["get_file_details", "search_files"], "google_docs": ["get_document"], "exa": ["answer"]}}
+{"user_input": "Create a summary of overnight updates from 8:00 PM yesterday to 8:00 AM today in IST. Check Gmail for important emails and ClickUp for mentions and assigned tasks. Organize findings into high priority and other items, then provide a comprehensive summary of all overnight activity.", "difficulty": 4, "required_tools": {"google_mail": ["list_messages"], "clickup": ["comments_get_task_comments", "comments_get_list_comments", "comments_get_view_comments", "tasks_get_list_tasks", "tasks_filter_team_tasks", "time_tracking_get_time_entries_within_date_range", "time_tracking_get_time_entry_history", "authorization_get_workspace_list", "spaces_get_details", "lists_get_list_details"]}}
+{"user_input": "Analyze the data in Google Sheet (ID: 1nnnCp3_IWcdHv4UVgXtwYF5wedxbqF4RIeyjN6mCKD8) and create 3-5 relevant charts and visualizations. Add pie charts, bar graphs, and other appropriate visualizations based on the data structure. Embed all charts directly into the sheet and provide the updated sheet link.", "difficulty": 4, "required_tools": {"google_sheet": ["create_spreadsheet", "get_spreadsheet_metadata", "batch_get_values_by_range", "append_dimensions", "insert_dimensions", "delete_sheet", "add_sheet", "delete_dimensions", "add_basic_chart", "add_table", "add_pie_chart", "clear_values", "update_values", "clear_basic_filter", "get_values", "discover_tables", "set_basic_filter", "analyze_table_schema", "copy_sheet_to_spreadsheet", "append_values", "batch_get_values_by_data_filter", "batch_clear_values", "format_cells"]}}
+{"user_input": "Create a 7-day learning plan for Python Programming. Research essential concepts and skills, create a detailed day-by-day plan with topics, goals, resources, and exercises. Compile the plan into a Google Doc and schedule daily emails at 8 AM starting today. Send Day 1 immediately to adit@agentr.dev and provide the Google Doc link.", "difficulty": 5, "required_tools": {"google_docs": ["get_document", "create_document", "insert_text"], "google_mail": ["send_email", "send_draft", "create_draft"], "exa": ["answer"]}}

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.15}/src/evals/evaluators.py

@@ -7,6 +7,8 @@ from langsmith.evaluation import EvaluationResult, run_evaluator
 from langsmith.schemas import Example, Run
 from openevals.llm import create_llm_as_judge
 
+from evals.prompts import CODEACT_EVALUATOR_PROMPT, CORRECTNESS_PROMPT
+
 
 @run_evaluator
 def exact_match_evaluator(run: Run, example: Example | None = None) -> EvaluationResult:
@@ -38,58 +40,6 @@ def exact_match_evaluator(run: Run, example: Example | None = None) -> Evaluatio
     return EvaluationResult(key="exact_match", score=score, comment=comment)
 
 
-CORRECTNESS_PROMPT = """You are an expert data labeler evaluating model outputs for correctness. Your task is to assign a score based on the following rubric:
-
-<Rubric>
-  A correct answer:
-  - Provides accurate and complete information
-  - Contains no factual errors
-  - Addresses all parts of the question
-  - Is logically consistent
-  - Uses precise and accurate terminology
-
-  When scoring, you should penalize:
-  - Factual errors or inaccuracies
-  - Incomplete or partial answers
-  - Misleading or ambiguous statements
-  - Incorrect terminology
-  - Logical inconsistencies
-  - Missing key information
-
-  Ignore the following:
-  - If the answer is not in the same language as the question.
-  - use the specifically requested tool, as the tool name can be different
-  - Do not penalize for incorrect third party data coming from the tool.
-</Rubric>
-
-<Instructions>
-  - Carefully read the input and output
-  - Check for factual accuracy and completeness
-  - Focus on correctness of information rather than style or verbosity
-  - If the user tool is not authorized, give a partial credit of `0.5`
-  - Give partial credit if tools and called correctly, but the data is incorrect from tools.
-</Instructions>
-
-<Reminder>
-  The goal is to evaluate factual correctness and completeness of the response.
-</Reminder>
-
-<input>
-{inputs}
-</input>
-
-<output>
-{outputs}
-</output>
-
-Use the reference outputs below to help you evaluate the correctness of the response:
-
-<reference_outputs>
-{reference_outputs}
-</reference_outputs>
-"""
-
-
 correctness_evaluator = create_llm_as_judge(
     prompt=CORRECTNESS_PROMPT,
     feedback_key="correctness",
@@ -103,8 +53,8 @@ trajectory_evaluator = create_trajectory_llm_as_judge(
 )
 
 
-codeact_evaluator = create_trajectory_llm_as_judge(
-    prompt=TRAJECTORY_ACCURACY_PROMPT,
+codeact_evaluator = create_llm_as_judge(
+    prompt=CODEACT_EVALUATOR_PROMPT,
     feedback_key="codeact_accuracy",
     model="anthropic:claude-4-sonnet-20250514",
 )

universal_mcp_agents-0.1.15/src/evals/prompts.py

@@ -0,0 +1,66 @@
+CORRECTNESS_PROMPT = """You are an expert data labeler evaluating model outputs for correctness. Your task is to assign a score based on the following rubric:
+
+<Rubric>
+  A correct answer:
+  - Provides accurate and complete information
+  - Contains no factual errors
+  - Addresses all parts of the question
+  - Is logically consistent
+  - Uses precise and accurate terminology
+
+  When scoring, you should penalize:
+  - Factual errors or inaccuracies
+  - Incomplete or partial answers
+  - Misleading or ambiguous statements
+  - Incorrect terminology
+  - Logical inconsistencies
+  - Missing key information
+
+  Ignore the following:
+  - If the answer is not in the same language as the question.
+  - use the specifically requested tool, as the tool name can be different
+  - Do not penalize for incorrect third party data coming from the tool.
+</Rubric>
+
+<Instructions>
+  - Carefully read the input and output
+  - Check for factual accuracy and completeness
+  - Focus on correctness of information rather than style or verbosity
+  - If the user tool is not authorized, give a partial credit of `0.5`
+  - Give partial credit if tools and called correctly, but the data is incorrect from tools.
+</Instructions>
+
+<Reminder>
+  The goal is to evaluate factual correctness and completeness of the response.
+</Reminder>
+
+<input>
+{inputs}
+</input>
+
+<output>
+{outputs}
+</output>
+
+Use the reference outputs below to help you evaluate the correctness of the response:
+
+<reference_outputs>
+{reference_outputs}
+</reference_outputs>
+"""
+
+CODEACT_EVALUATOR_PROMPT = """
+You are a code execution evaluator. You will be given the entire run of an agent, starting with a human input task, the intermediate steps taken, and the final output of the agent given to the user. These steps will contain code written by the agent to solve the problem as well as its outputs. Your job is to check ONLY if the code executes correctly.
+Keep in mind that the agent has access to tools like- ai_classify, call_llm, creative_writer, data_extractor. These calls are to be treated as valid if they run without errors.
+These are the only criteria you should evaluate-
+
+<Rubric>
+- The code written by the agent in tool calls should be syntactically correct and use existing objects.
+- The code outputs should not have an error or empty/unexpected outputs
+</Rubric>
+If either of the above are not satisfied, you should give 0.
+
+<Reminder>
+You must not judge whether the code is helpful to the task or not, only if the code itself is correct or not.
+</Reminder>
+"""

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.15}/src/evals/run.py

@@ -11,6 +11,7 @@ from universal_mcp.agentr.registry import AgentrRegistry
 
 from evals.dataset import load_dataset
 from evals.evaluators import (
+    codeact_evaluator,
     correctness_evaluator,
     exact_match_evaluator,
     tool_node_evaluator,
@@ -62,12 +63,12 @@ async def agent_runner(agent_name: str, inputs: dict) -> dict:
     registry = AgentrRegistry(client=client) if agent_name != "simple" else None
     common_params = {
         "instructions": f"You are a helpful assistant. Keep your responses short and concise. Do not provide with any explanation. The current date and time is {current_date_time}",
-        "model": "anthropic/claude-4-sonnet-20250514",
+        "model": "azure/gpt-4.1",
         "registry": registry,
         "tools": inputs.get("tools", {}),
     }
     agent = get_agent(agent_name)(name=agent_name, **common_params)
-    result = await agent.invoke(user_input=inputs["user_input"])
+    result = await agent.invoke(user_input=inputs["user_input"], thread_id="evals")
     messages = messages_to_list(result["messages"])
     return_result = {"output": messages}
     if "tool_config" in result:

{universal_mcp_agents-0.1.13 → universal_mcp_agents-0.1.15}/src/tests/test_agents.py

@@ -145,6 +145,7 @@ class MockToolRegistry(ToolRegistry):
         self,
         query: str,
         limit: int = 10,
+        distance_threshold: float = 0.7,
     ) -> list[dict[str, Any]]:
         """
         Search for apps by a query.
@@ -167,6 +168,7 @@ class MockToolRegistry(ToolRegistry):
         query: str,
         limit: int = 10,
         app_id: str | None = None,
+        distance_threshold: float = 0.8,
     ) -> list[dict[str, Any]]:
         """
         Search for tools by a query.
@@ -175,11 +177,23 @@ class MockToolRegistry(ToolRegistry):
         brittle keyword search.
         """
         if not app_id:
-            return []
-
-        # Return all tools for the given app, letting the LLM choose.
+            # General search
+            all_tools = []
+            for current_app_id, tools in self._tools.items():
+                for tool in tools:
+                    tool_with_app_id = tool.copy()
+                    tool_with_app_id["id"] = f"{current_app_id}__{tool['name']}"
+                    all_tools.append(tool_with_app_id)
+            return all_tools[:limit]
+
+        # App-specific search
         all_app_tools = self._tools.get(app_id, [])
-        return all_app_tools[:limit]
+        tools_with_app_id = []
+        for tool in all_app_tools:
+            tool_with_app_id = tool.copy()
+            tool_with_app_id["id"] = f"{app_id}__{tool['name']}"
+            tools_with_app_id.append(tool_with_app_id)
+        return tools_with_app_id[:limit]
 
     async def export_tools(
         self,
@@ -217,17 +231,6 @@ class TestToolFinderGraph:
     def registry(self):
         return MockToolRegistry()
 
-    def _get_tool_config_from_plan(self, plan: list[dict[str, Any]]) -> dict[str, list[str]]:
-        """
-        Helper function to convert a consolidated execution plan to a tool_config dict.
-        MODIFIED: This now correctly handles the already-consolidated plan from the graph.
-        """
-        if not plan:
-            return {}
-
-        config = {step["app_id"]: step["tool_ids"] for step in plan if step.get("app_id") and step.get("tool_ids")}
-        return config
-
     @pytest.mark.asyncio
     async def test_simple_case(self, llm, registry):
         """Test Case 1: Simple task requiring a single app and tool."""
@@ -237,9 +240,7 @@ class TestToolFinderGraph:
             {"original_task": task, "messages": [HumanMessage(content=task)], "decomposition_attempts": 0}
         )
 
-        plan = final_state.get("execution_plan")
-
-        tool_config = self._get_tool_config_from_plan(plan)
+        tool_config = final_state.get("execution_plan")
 
         # FIX: Assert against the correct, hyphenated app ID.
         assert "google_mail" in tool_config
@@ -254,10 +255,8 @@ class TestToolFinderGraph:
             {"original_task": task, "messages": [HumanMessage(content=task)], "decomposition_attempts": 0}
         )
 
-        plan = final_state.get("execution_plan")
-        assert plan, "Execution plan should not be empty"
-
-        tool_config = self._get_tool_config_from_plan(plan)
+        tool_config = final_state.get("execution_plan")
+        assert tool_config, "Execution plan should not be empty"
 
         assert "github" in tool_config
         assert "create_issue" in tool_config["github"]
@@ -275,7 +274,7 @@ class TestToolFinderGraph:
         plan = final_state.get("execution_plan")
         assert not plan
         last_message = final_state.get("messages", [])[-1].content
-        assert "unable to create a complete plan" in last_message.lower()
+        assert "could not create a final plan" in last_message.lower()
 
 
 @pytest.mark.parametrize(
@@ -312,19 +311,20 @@ class TestAgents:
         await agent.ainit()
         # Invoke the agent graph to get the final state
         final_state = await agent.invoke(
-            user_input=task,
+            user_input={"userInput": task} if agent.name == "Test builder" else task,
             thread_id=thread_id,
         )
 
         # Extract the content of the last message
-        final_messages = final_state.get("messages", [])
-        assert final_messages, "The agent should have produced at least one message."
-        last_message = final_messages[-1]
+        if agent.name != "Test builder":
+            final_messages = final_state.get("messages", [])
+            assert final_messages, "The agent should have produced at least one message."
+            last_message = final_messages[-1]
 
-        final_response = last_message.content if hasattr(last_message, "content") else str(last_message)
+            final_response = last_message.content if hasattr(last_message, "content") else str(last_message)
 
-        assert final_response is not None, "The final response should not be None."
-        assert final_response != "", "The final response should not be an empty string."
+            assert final_response is not None, "The final response should not be None."
+            assert final_response != "", "The final response should not be an empty string."
 
 
 class TestAgentBuilder:
@@ -344,8 +344,9 @@ class TestAgentBuilder:
     async def test_create_agent(self, agent_builder: BuilderAgent):
         """Test case for creating an agent with the builder."""
         task = "Send a daily email to manoj@agentr.dev with daily agenda of the day"
+        thread_id = "test-thread-create-agent"
 
-        result = await agent_builder.invoke(task)
+        result = await agent_builder.invoke(thread_id=thread_id, user_input={"userInput": task})
 
         assert "generated_agent" in result
         generated_agent = result["generated_agent"]