universal-mcp-agents 0.1.9__py3-none-any.whl → 0.1.10__py3-none-any.whl

universal_mcp/agents/bigtoolcache/__init__.py

@@ -1,53 +1,62 @@
+from datetime import UTC, datetime
+
 from langgraph.checkpoint.base import BaseCheckpointSaver
+from loguru import logger
+from universal_mcp.tools.registry import ToolRegistry
+from universal_mcp.types import ToolConfig, ToolFormat
 
 from universal_mcp.agents.base import BaseAgent
 from universal_mcp.agents.llm import load_chat_model
-from universal_mcp.logger import logger
-from universal_mcp.tools.registry import ToolRegistry
 
 from .graph import build_graph
 from .prompts import SYSTEM_PROMPT
+from .tools import create_meta_tools
 
 
 class BigToolAgentCache(BaseAgent):
     def __init__(
         self,
-        name: str,
-        instructions: str,
-        model: str,
         registry: ToolRegistry,
+        name: str = "Wingman",
+        instructions: str = "",
+        model: str = "anthropic/claude-4-sonnet-20250514",
+        tools: ToolConfig | None = None,
         memory: BaseCheckpointSaver | None = None,
         **kwargs,
     ):
-        # Combine the base system prompt with agent-specific instructions
-        full_instructions = f"{SYSTEM_PROMPT}\n\n**User Instructions:**\n{instructions}"
-        super().__init__(name, full_instructions, model, memory, **kwargs)
+        super().__init__(name, instructions, model, memory, **kwargs)
 
-        self.registry = registry
-        self.llm = load_chat_model(self.model)
+        self._tool_registry = registry
+        self._tools = tools or {}
+        if "ui" not in self._tools:
+            self._tools["ui"] = ["create_table"]
         self.recursion_limit = kwargs.get("recursion_limit", 10)
 
-        logger.info(
-            f"BigToolAgent '{self.name}' initialized with model '{self.model}'."
+        logger.info(f"BigToolAgent '{self.name}' initialized with model '{self.model}'.")
+
+    def _build_system_message(self):
+        return SYSTEM_PROMPT.format(
+            name=self.name,
+            instructions=f"**User Instructions:**\n{self.instructions}" if len(self.instructions) > 0 else "",
+            system_time=datetime.now(tz=UTC).isoformat(),
         )
 
     async def _build_graph(self):
-        """Build the bigtool agent graph using the existing create_agent function."""
-        logger.info(f"Building graph for BigToolAgent '{self.name}'...")
+        """Build the LangGraph workflow"""
         try:
+            default_tools = await self._tool_registry.export_tools(self._tools, ToolFormat.LANGCHAIN)
+            meta_tools = create_meta_tools(self._tool_registry)
             graph_builder = build_graph(
-                tool_registry=self.registry,
-                llm=self.llm,
+                registry=self._tool_registry,
+                base_model=load_chat_model(self.model),
+                system_prompt=self._build_system_message(),
+                default_tools=default_tools,
+                meta_tools=meta_tools,
             )
-
             compiled_graph = graph_builder.compile(checkpointer=self.memory)
-            logger.info("Graph built and compiled successfully.")
             return compiled_graph
         except Exception as e:
-            logger.error(
-                f"Error building graph for BigToolAgentCache '{self.name}': {e}"
-            )
-            raise
+            raise e
 
     @property
     def graph(self):

universal_mcp/agents/bigtoolcache/__main__.py

@@ -1,16 +1,13 @@
 import asyncio
 
 from loguru import logger
-
 from universal_mcp.agentr.registry import AgentrRegistry
+
 from universal_mcp.agents.bigtoolcache import BigToolAgentCache
 
 
 async def main():
     agent = BigToolAgentCache(
-        name="bigtoolcache",
-        instructions="You are a helpful assistant that can use tools to help the user.",
-        model="azure/gpt-4.1",
         registry=AgentrRegistry(),
     )
     async for event in agent.stream(

universal_mcp/agents/bigtoolcache/agent.py

@@ -1,12 +1,10 @@
 from universal_mcp.agentr.registry import AgentrRegistry
+
 from universal_mcp.agents.bigtoolcache import BigToolAgentCache
 
 
 async def agent():
     agent_object = await BigToolAgentCache(
-        name="BigTool Agent Cache version",
-        instructions="You are a helpful assistant that can use various tools to complete tasks.",
-        model="anthropic/claude-4-sonnet-20250514",
         registry=AgentrRegistry(),
     )._build_graph()
     return agent_object

universal_mcp/agents/bigtoolcache/graph.py

@@ -1,204 +1,114 @@
 import json
-from datetime import UTC, datetime
-from typing import Any, Literal, TypedDict, cast
+from typing import Literal, cast
 
+from dotenv import load_dotenv
 from langchain_anthropic import ChatAnthropic
 from langchain_core.language_models import BaseChatModel
-from langchain_core.messages import AIMessage, ToolMessage
-from langchain_core.tools import tool
+from langchain_core.messages import AIMessage, SystemMessage, ToolMessage
+from langchain_core.tools import BaseTool
 from langgraph.graph import StateGraph
-from langgraph.runtime import Runtime
 from langgraph.types import Command
-
-from universal_mcp.agents.bigtoolcache.context import Context
-from universal_mcp.agents.bigtoolcache.prompts import TOOLS_LIST
-from universal_mcp.agents.bigtoolcache.state import State
-from universal_mcp.logger import logger
+from loguru import logger
 from universal_mcp.tools.registry import ToolRegistry
 from universal_mcp.types import ToolFormat
 
-
-class ToolSelectionOutput(TypedDict):
-    connected_tool_ids: list[str]
-    unconnected_tool_ids: list[str]
-
-
-def build_graph(tool_registry: ToolRegistry, llm: BaseChatModel):
-    @tool
-    async def search_tools(queries: list[str]) -> str:
-        """Search tools for a given list of queries
-        Each single query should be atomic (doable with a single tool).
-        For tasks requiring multiple tools, add separate queries for each subtask"""
-        logger.info(f"Searching tools for queries: '{queries}'")
-        messages = [
-            {
-                "role": "system",
-                "content": [
-                    {
-                        "type": "text",
-                        "text": TOOLS_LIST,
-                        "cache_control": {"type": "ephemeral", "ttl": "1h"},
-                    }
-                ],
-            },
-            {"role": "user", "content": str(queries)},
-        ]
-        response = llm.with_structured_output(ToolSelectionOutput).invoke(messages)
-        response_text = f"Connected tools: {response['connected_tool_ids']}\nUnconnected tools: {response['unconnected_tool_ids']}"
-        return response_text
-
-    @tool
-    async def load_tools(tool_ids: list[str]) -> list[dict[str, Any]]:
-        """Load the tools for the given tool ids. Returns the tool name, description, parameters schema, and output schema. A tool id is made up using the app_id and the tool_name, attached by double underscore (__). e.g. google_mail__send_email"""
-        temp_manager = tool_registry.tool_manager
-        temp_manager.clear_tools()
-        await tool_registry.export_tools(tool_ids, format=ToolFormat.NATIVE)
-        tool_details = []
-        for tool_id in tool_ids:
-            tool = temp_manager.get_tool(tool_id)
-            tool_details.append(
-                {
-                    "name": tool.name,
-                    "description": tool.description,
-                    "parameters_schema": tool.parameters,
-                    "output_schema": tool.output_schema,
-                }
+from .state import State
+from .tools import get_valid_tools
+
+load_dotenv()
+
+
+def build_graph(
+    registry: ToolRegistry,
+    base_model: BaseChatModel,
+    system_prompt: str,
+    default_tools: list[BaseTool],
+    meta_tools: dict[str, BaseTool],
+):
+    """Build the LangGraph workflow"""
+
+    async def agent_node(state: State) -> Command[Literal["execute_tools"]]:
+        """Main agent reasoning node"""
+
+        # Combine meta tools with currently loaded tools
+        if len(state["selected_tool_ids"]) > 0:
+            current_tools = await registry.export_tools(tools=state["selected_tool_ids"], format=ToolFormat.LANGCHAIN)
+        else:
+            current_tools = []
+        all_tools = [meta_tools["search_tools"], meta_tools["load_tools"], meta_tools.get("web_search")] + default_tools + current_tools
+
+        # Remove duplicates based on tool name
+        seen_names = set()
+        unique_tools = []
+        for tool in all_tools:
+            if tool.name not in seen_names:
+                seen_names.add(tool.name)
+                unique_tools.append(tool)
+
+        if isinstance(base_model, ChatAnthropic):
+            model_with_tools = base_model.bind_tools(
+                unique_tools,
+                tool_choice="auto",
+                parallel_tool_calls=False,
+                cache_control={"type": "ephemeral", "ttl": "1h"},
             )
-        return tool_details
-
-    @tool
-    async def call_tool(tool_id: str, tool_args: dict[str, Any]) -> Any:
-        """Call the tool with the given id and arguments."""
-        return await tool_registry.call_tool(tool_id, tool_args)
-
-    async def call_model(
-        state: State, runtime: Runtime[Context]
-    ) -> Command[Literal["select_tools", "call_tools"]]:
-        logger.info("Calling model...")
-        try:
-            system_message = runtime.context.system_prompt.format(
-                system_time=datetime.now(tz=UTC).isoformat()
+        else:
+            model_with_tools = base_model.bind_tools(
+                unique_tools,
+                tool_choice="auto",
+                parallel_tool_calls=False,
             )
-            messages = [
-                {
-                    "role": "system",
-                    "content": [
-                        {
-                            "type": "text",
-                            "text": system_message,
-                            "cache_control": {"type": "ephemeral", "ttl": "1h"},
-                        }
-                    ],
-                },
-                *state["messages"],
-            ]
-
-            model = llm
-
-            if isinstance(model, ChatAnthropic):
-                model_with_tools = model.bind_tools(
-                    [search_tools, load_tools, call_tool],
-                    tool_choice="auto",
-                    cache_control={"type": "ephemeral", "ttl": "1h"},
-                )
-            else:
-                model_with_tools = model.bind_tools(
-                    [search_tools, load_tools, call_tool], tool_choice="auto"
-                )
-            response = cast(AIMessage, await model_with_tools.ainvoke(messages))
-
-            if response.tool_calls:
-                logger.info(
-                    f"Model responded with {len(response.tool_calls)} tool calls."
-                )
-                if len(response.tool_calls) > 1:
-                    raise Exception(
-                        "Not possible in Claude with llm.bind_tools(tools=tools, tool_choice='auto')"
-                    )
-                tool_call = response.tool_calls[0]
-                if tool_call["name"] == search_tools.name:
-                    logger.info("Model requested to select tools.")
-                    return Command(goto="select_tools", update={"messages": [response]})
-                elif tool_call["name"] == load_tools.name:
-                    logger.info("Model requested to load tools.")
-                    tool_details = await load_tools.ainvoke(input=tool_call["args"])
-                    tool_msg = ToolMessage(
-                        f"Loaded tools. {tool_details}", tool_call_id=tool_call["id"]
-                    )
-                    selected_tool_ids = tool_call["args"]["tool_ids"]
-                    logger.info(f"Loaded tools: {selected_tool_ids}")
-                    return Command(
-                        goto="call_model",
-                        update={
-                            "messages": [response, tool_msg],
-                            "selected_tool_ids": selected_tool_ids,
-                        },
-                    )
-                elif tool_call["name"] == call_tool.name:
-                    logger.info("Model requested to call tool.")
-                    return Command(goto="call_tools", update={"messages": [response]})
-                return Command(goto="call_tools", update={"messages": [response]})
+
+        # Get response from model
+        messages = [SystemMessage(content=system_prompt), *state["messages"]]
+        response = cast(AIMessage, await model_with_tools.ainvoke(messages))
+
+        if response.tool_calls:
+            return Command(goto="execute_tools", update={"messages": [response]})
+        else:
+            return Command(update={"messages": [response], "model_with_tools": model_with_tools})
+
+    async def execute_tools_node(state: State) -> Command[Literal["agent"]]:
+        """Execute tool calls"""
+        last_message = state["messages"][-1]
+        tool_calls = last_message.tool_calls if isinstance(last_message, AIMessage) else []
+
+        tool_messages = []
+        new_tool_ids = []
+
+        for tool_call in tool_calls:
+            if tool_call["name"] == "load_tools":  # Handle load_tools separately
+                valid_tools = await get_valid_tools(tool_ids=tool_call["args"]["tool_ids"], registry=registry)
+                new_tool_ids.extend(valid_tools)
+                # Create tool message response
+                tool_result=f"Successfully loaded {len(valid_tools)} tools: {valid_tools}"
+            elif tool_call["name"] == "search_tools":
+                tool_result = await meta_tools["search_tools"].ainvoke(tool_call["args"])
+            elif tool_call["name"] == "web_search":
+                tool_result = await meta_tools["web_search"].ainvoke(tool_call["args"])
             else:
-                logger.info("Model responded with a message, ending execution.")
-                return Command(update={"messages": [response]})
-        except Exception as e:
-            logger.error(f"Error in call_model: {e}")
-            raise
-
-    async def select_tools(
-        state: State, runtime: Runtime[Context]
-    ) -> Command[Literal["call_model"]]:
-        logger.info("Selecting tools...")
-        try:
-            tool_call = state["messages"][-1].tool_calls[0]
-            searched_tools = await search_tools.ainvoke(input=tool_call["args"])
-            tool_msg = ToolMessage(
-                f"Available tools: {searched_tools}", tool_call_id=tool_call["id"]
+                # Load tools first
+                await registry.export_tools([tool_call["name"]], ToolFormat.LANGCHAIN)
+                tool_result = await registry.call_tool(tool_call["name"], tool_call["args"])
+            tool_message = ToolMessage(
+                content=json.dumps(tool_result),
+                name=tool_call["name"],
+                tool_call_id=tool_call["id"],
             )
-            return Command(goto="call_model", update={"messages": [tool_msg]})
-        except Exception as e:
-            logger.error(f"Error in select_tools: {e}")
-            raise
-
-    async def call_tools(state: State) -> Command[Literal["call_model"]]:
-        logger.info("Calling tools...")
-        outputs = []
-        recent_tool_ids = []
-        tool_call = state["messages"][-1].tool_calls[0]
-        tool_id = tool_call["args"]["tool_id"]
-        tool_args = tool_call["args"]["tool_args"]
-        logger.info(f"Executing tool: {tool_id} with args: {tool_args}")
-        try:
-            await tool_registry.export_tools([tool_id], ToolFormat.LANGCHAIN)
-            tool_result = await call_tool.ainvoke(
-                input={"tool_id": tool_id, "tool_args": tool_args}
-            )
-            logger.info(f"Tool '{tool_id}' executed successfully.")
-            outputs.append(
-                ToolMessage(
-                    content=json.dumps(tool_result),
-                    name=tool_id,
-                    tool_call_id=tool_call["id"],
-                ))
-            recent_tool_ids.append(tool_call["name"])
-        except Exception as e:
-            logger.error(f"Error executing tool '{tool_call['name']}': {e}")
-            outputs.append(
-                ToolMessage(
-                    content=json.dumps("Error: " + str(e)),
-                    name=tool_call["name"],
-                    tool_call_id=tool_call["id"],
-                )
-            )
-        return Command(
-            goto="call_model",
-            update={"messages": outputs, "selected_tool_ids": recent_tool_ids},
-        )
-
-    builder = StateGraph(State, context_schema=Context)
-
-    builder.add_node(call_model)
-    builder.add_node(select_tools)
-    builder.add_node(call_tools)
-    builder.set_entry_point("call_model")
-    return builder
+            tool_messages.append(tool_message)
+
+        return Command(goto="agent", update={"messages": tool_messages, "selected_tool_ids": new_tool_ids})
+
+
+
+    # Define the graph
+    workflow = StateGraph(State)
+
+    # Add nodes
+    workflow.add_node("agent", agent_node)
+    workflow.add_node("execute_tools", execute_tools_node)
+
+    # Set entry point
+    workflow.set_entry_point("agent")
+
+    return workflow

universal_mcp/agents/bigtoolcache/prompts.py

@@ -1,41 +1,17 @@
 """Default prompts used by the agent."""
 
-from pathlib import Path
-
-
-def load_tools_from_file():
-    """Load tools from the generated text file."""
-    # Get the directory where this file is located
-    current_dir = Path(__file__).parent
-
-    tools_file = current_dir / "tools_important.txt"
-    if not tools_file.exists():
-        tools_file = current_dir / "tools_all.txt"
-
-    if tools_file.exists():
-        with open(tools_file, encoding="utf-8") as f:
-            return f.read()
-    else:
-        return "No tools file found. Please run tool_retrieve.py to generate the tools list."
-
-
-SYSTEM_PROMPT = """You are a helpful AI assistant.
+SYSTEM_PROMPT = """You are a helpful AI assistant, called {name}.
 
 **Core Directives:**
-1.  **Always Use Tools for Tasks:** For any user request that requires an action (e.g., sending an email, searching for information, creating an event), you MUST use a tool. Do not answer from your own knowledge or refuse a task if a tool might exist for it.
-2.  **First Step is ALWAYS `search_tools`:** Before you can use any other tool, you MUST first call the `search_tools` function to find the right tools for the user's request. This is your mandatory first action. You must not use the same/similar query multiple times in the list. The list should have multiple queries only if the task has clearly different sub-tasks.
-3.  **Load Tools:** After looking at the output of `search_tools`, you MUST call the `load_tools` function to load only the tools you want to use. Use your judgement to eliminate irrelevant apps that came up just because of semantic similarity. However, sometimes, multiple apps might be relevant for the same task. Prefer connected apps over unconnected apps while breaking a tie. If more than one relevant app (or none of the relevant apps) are connected, you must ask the user to choose the app. In case the user asks you to use an app that is not connected, call the apps tools normally. The tool will return a link for connecting that you should pass on to the user.
-4.  **Call Tools:** After loading the tools, you MUST call the `call_tool` function to call the tools you want to use. You must call the tool with the correct arguments. You can only call the tool once you have loaded it.
-5.  **Strictly Follow the Process:** Your only job in your first turn is to analyze the user's request and call `search_tools` with a concise query describing the core task. Do not engage in conversation.
+1.  **Always Use Tools for Tasks:** For any user request that requires an action (e.g., sending an email, searching for information, creating an event, displaying a chart), you MUST use a tool. Do not refuse a task if a tool might exist for it.
 
-System time: {system_time}
-"""
+2. Check if your existing tools or knowledge can handle the user's request. If they can, use them. If they cannot, you must call the `search_tools` function to find the right tools for the user's request.You must not use the same/similar query multiple times in the list. The list should have multiple queries only if the task has clearly different sub-tasks. If you do not find any specific relevant tools, use the pre-loaded generic tools.
 
+3.  **Load Tools:** After looking at the output of `search_tools`, you MUST call the `load_tools` function to load only the tools you want to use. Provide the full tool ids, not just the app names. Use your judgement to eliminate irrelevant apps that came up just because of semantic similarity. However, sometimes, multiple apps might be relevant for the same task. Prefer connected apps over unconnected apps while breaking a tie. If more than one relevant app (or none of the relevant apps) are connected, you must ask the user to choose the app. In case the user asks you to use an app that is not connected, call the apps tools normally. The tool will return a link for connecting that you should pass on to the user.
 
-TOOLS_LIST = f""" This is the list of all the tools available to you:
-{load_tools_from_file()}
+4.  **Strictly Follow the Process:** Your only job in your first turn is to analyze the user's request and answer using existing tools/knowledge or `search_tools` with a concise query describing the core task. Do not engage in conversation, or extend the conversation beyond the user's request.
 
-You will be provided a list of queries (which may be similar or different from each other). Your job is to select the relavent tools for the user's request. sometimes, multiple apps might be relevant for the same task. Prefer connected apps over unconnected apps while breaking a tie. If more than one relevant app (or none of the relevant apps) are connected, you must return both apps tools. If the query specifically asks you to use an app that is not connected, return the tools for that app, they can still be connected by the user.
+{instructions}
 
-You have to return the tool_ids by constructing the tool_id from the app_id and the tool_name, attached by double underscore (__). e.g. google_mail__send_email
+System time: {system_time}
 """

universal_mcp/agents/bigtoolcache/tools.py

@@ -0,0 +1,141 @@
+import asyncio
+from collections import defaultdict
+from typing import Any
+
+from langchain_core.tools import tool
+from universal_mcp.tools.registry import ToolRegistry
+from universal_mcp.types import ToolFormat
+
+
+def create_meta_tools(tool_registry: ToolRegistry) -> dict[str, Any]:
+    """Create the meta tools for searching and loading tools"""
+
+    @tool
+    async def search_tools(queries: list[str]) -> str:
+        """Search for relevant tools given list of queries.
+        Each single query should be atomic (doable with a single tool).
+        For tasks requiring multiple tools, add separate queries for each subtask"""
+        try:
+            # Fetch all connections
+            connections = await tool_registry.list_connected_apps()
+            connected_apps = {connection["app_id"] for connection in connections}
+
+            # Use defaultdict to avoid key existence checks
+            app_tools = defaultdict(list)
+
+            # Process all queries concurrently
+            search_tasks = []
+            for query in queries:
+                search_tasks.append(_search_query_tools(query))
+
+            query_results = await asyncio.gather(*search_tasks)
+
+            # Aggregate results with limit per app
+            for tools_list in query_results:
+                for tool in tools_list:
+                    app = tool["id"].split("__")[0]
+                    if len(app_tools[app]) < 5:
+                        app_tools[app].append(f"{tool['id']}: {tool['description']}")
+
+            # Build result string efficiently
+            result_parts = []
+            for app, tools in app_tools.items():
+                app_status = "connected" if app in connected_apps else "NOT connected"
+                result_parts.append(f"Tools from {app} (status: {app_status} by user):")
+                for tool in tools:
+                    result_parts.append(f" - {tool}")
+                result_parts.append("")  # Empty line between apps
+
+            result_parts.append("Call load_tools to select the required tools only.")
+            return "\n".join(result_parts)
+
+        except Exception as e:
+            return f"Error: {e}"
+
+    async def _search_query_tools(query: str) -> list[dict]:
+        """Helper function to search apps and tools for a single query."""
+        # Start both searches concurrently
+        tools_search_task = tool_registry.search_tools(query, limit=10)
+        apps_search_task = tool_registry.search_apps(query, limit=4)
+
+        # Wait for both to complete
+        tools_from_general_search, apps_list = await asyncio.gather(tools_search_task, apps_search_task)
+
+        # Create tasks for searching tools from each app
+        app_tool_tasks = [tool_registry.search_tools(query, limit=5, app_id=app["id"]) for app in apps_list]
+
+        # Wait for all app-specific tool searches to complete
+        app_tools_results = await asyncio.gather(*app_tool_tasks)
+
+        # Combine all results
+        tools_list = list(tools_from_general_search)
+        for app_tools in app_tools_results:
+            tools_list.extend(app_tools)
+
+        return tools_list
+
+    @tool
+    async def load_tools(tool_ids: list[str]) -> str:
+        """Load specific tools by their IDs for use in subsequent steps.
+
+        Args:
+            tool_ids: Tool ids in the form 'app__tool'. Example: 'google_mail__send_email'
+
+        Returns:
+            Confirmation message about loaded tools
+        """
+        return f"Successfully loaded {len(tool_ids)} tools: {tool_ids}"
+
+    @tool
+    async def web_search(query: str) -> str:
+        """Search the web for the given query. Returns the search results. Do not use for app-specific searches (for example, reddit or linkedin searches should be done using the app's tools)"""
+        await tool_registry.export_tools(["exa__search_with_filters"], ToolFormat.LANGCHAIN)
+        response = await tool_registry.call_tool(
+            "exa__search_with_filters", {"query": query, "contents": {"summary": True}}
+        )
+        return response
+
+    return {"search_tools": search_tools, "load_tools": load_tools, "web_search": web_search}
+
+
+async def get_valid_tools(tool_ids: list[str], registry: ToolRegistry) -> list[str]:
+    correct, incorrect = [], []
+    app_tool_list: dict[str, set[str]] = {}
+
+    # Group tool_ids by app for fewer registry calls
+    app_to_tools: dict[str, list[tuple[str, str]]] = {}
+    for tool_id in tool_ids:
+        if "__" not in tool_id:
+            incorrect.append(tool_id)
+            continue
+        app, tool_name = tool_id.split("__", 1)
+        app_to_tools.setdefault(app, []).append((tool_id, tool_name))
+
+    # Fetch all apps concurrently
+    async def fetch_tools(app: str):
+        try:
+            tools_dict = await registry.list_tools(app)
+            return app, {tool_unit["name"] for tool_unit in tools_dict}
+        except Exception:
+            return app, None
+
+    results = await asyncio.gather(*(fetch_tools(app) for app in app_to_tools))
+
+    # Build map of available tools per app
+    for app, tools in results:
+        if tools is not None:
+            app_tool_list[app] = tools
+
+    # Validate tool_ids
+    for app, tool_entries in app_to_tools.items():
+        available = app_tool_list.get(app)
+        if available is None:
+            incorrect.extend(tool_id for tool_id, _ in tool_entries)
+            continue
+        for tool_id, tool_name in tool_entries:
+            if tool_name in available:
+                correct.append(tool_id)
+            else:
+                incorrect.append(tool_id)
+
+    return correct