tooluniverse 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

tooluniverse/execute_function.py

@@ -1000,9 +1000,9 @@ class ToolUniverse:
         """
         return copy.deepcopy(self.all_tools)
 
-    def list_built_in_tools(self, mode="config"):
+    def list_built_in_tools(self, mode="config", scan_all=False):
         """
-        List all built-in tool categories and their statistics with two different modes.
+        List all built-in tool categories and their statistics with different modes.
 
         This method provides a comprehensive overview of all available tools in the ToolUniverse,
         organized by categories. It reads directly from the default tool files to gather statistics,
@@ -1012,39 +1012,62 @@ class ToolUniverse:
             mode (str, optional): Organization mode for tools. Defaults to 'config'.
                 - 'config': Organize by config file categories (original behavior)
                 - 'type': Organize by tool types (implementation classes)
+                - 'list_name': Return a list of all tool names
+                - 'list_spec': Return a list of all tool specifications
+            scan_all (bool, optional): Whether to scan all JSON files in data directory recursively.
+                If True, scans all JSON files in data/ and its subdirectories.
+                If False (default), uses predefined tool file mappings.
 
         Returns:
-            dict: A dictionary containing tool statistics with the following structure:
-
-                {
-                    'categories': {
-                        'category_name': {
-                            'count': int,          # Number of tools in this category
-                            'tools': list          # List of tool names (only when mode='type')
-                        },
-                        ...
-                    },
-                    'total_categories': int,        # Total number of tool categories
-                    'total_tools': int,            # Total number of unique tools
-                    'mode': str,                   # The mode used for organization
-                    'summary': str                 # Human-readable summary of statistics
-                }
+            dict or list:
+                - For 'config' and 'type' modes: A dictionary containing tool statistics
+                - For 'list_name' mode: A list of all tool names
+                - For 'list_spec' mode: A list of all tool specifications
 
         Example:
             >>> tool_universe = ToolUniverse()
-            >>> # Group by config file categories
+            >>> # Group by config file categories (predefined files only)
             >>> stats = tool_universe.list_built_in_tools(mode='config')
-            >>> # Group by tool types
-            >>> stats = tool_universe.list_built_in_tools(mode='type')
+            >>> # Scan all JSON files in data directory recursively
+            >>> stats = tool_universe.list_built_in_tools(mode='config', scan_all=True)
+            >>> # Get all tool names from all JSON files
+            >>> tool_names = tool_universe.list_built_in_tools(mode='list_name', scan_all=True)
 
         Note:
             - This method reads directly from tool files and works without calling load_tools()
             - Tools are deduplicated across categories, so the same tool won't be counted multiple times
-            - The summary is automatically printed to console when this method is called
+            - The summary is automatically printed to console when this method is called (except for list_name and list_spec modes)
+            - When scan_all=True, all JSON files in data/ and subdirectories are scanned
         """
-        if mode not in ["config", "type"]:
-            raise ValueError("Mode must be either 'config' or 'type'")
+        if mode not in ["config", "type", "list_name", "list_spec"]:
+            raise ValueError(
+                "Mode must be one of: 'config', 'type', 'list_name', 'list_spec'"
+            )
 
+        # For list_name and list_spec modes, we can return early with just the data
+        if mode in ["list_name", "list_spec"]:
+            all_tools = []
+            all_tool_names = set()  # For deduplication across categories
+
+            if scan_all:
+                # Scan all JSON files in data directory recursively
+                all_tools, all_tool_names = self._scan_all_json_files()
+            else:
+                # Use predefined tool files (original behavior)
+                all_tools, all_tool_names = self._scan_predefined_files()
+
+            # Deduplicate tools by name
+            unique_tools = {}
+            for tool in all_tools:
+                if tool["name"] not in unique_tools:
+                    unique_tools[tool["name"]] = tool
+
+            if mode == "list_name":
+                return sorted(list(unique_tools.keys()))
+            elif mode == "list_spec":
+                return list(unique_tools.values())
+
+        # Original logic for config and type modes
         result = {
             "categories": {},
             "total_categories": 0,
@@ -1053,58 +1076,43 @@ class ToolUniverse:
             "summary": "",
         }
 
-        all_tool_names = set()  # For deduplication across categories
-        all_tools = []  # Store all tools for type-based grouping
-
-        # Read tools from each category file
-        for category, file_path in self.tool_files.items():
-            try:
-                # Read the JSON file for this category
-                tools_in_category = read_json_list(file_path)
-                all_tools.extend(tools_in_category)
+        if scan_all:
+            # Scan all JSON files in data directory recursively
+            all_tools, all_tool_names = self._scan_all_json_files()
 
-                if mode == "config":
-                    tool_names = [tool["name"] for tool in tools_in_category]
-                    result["categories"][category] = {"count": len(tool_names)}
+            # For config mode with scan_all, organize by file names
+            if mode == "config":
+                file_tools_map = {}
+                for tool in all_tools:
+                    # Get the source file for this tool (we need to track this)
+                    # For now, we'll organize by tool type as a fallback
+                    tool_type = tool.get("type", "Unknown")
+                    if tool_type not in file_tools_map:
+                        file_tools_map[tool_type] = []
+                    file_tools_map[tool_type].append(tool)
+
+                for category, tools in file_tools_map.items():
+                    result["categories"][category] = {"count": len(tools)}
+        else:
+            # Use predefined tool files (original behavior)
+            all_tools, all_tool_names = self._scan_predefined_files()
 
-                # Add to global set for deduplication
-                all_tool_names.update([tool["name"] for tool in tools_in_category])
+            # Read tools from each category file
+            for category, file_path in self.tool_files.items():
+                try:
+                    # Read the JSON file for this category
+                    tools_in_category = read_json_list(file_path)
 
-            except Exception as e:
-                warning(
-                    f"Warning: Could not read tools from {category} ({file_path}): {e}"
-                )
-                if mode == "config":
-                    result["categories"][category] = {"count": 0}
+                    if mode == "config":
+                        tool_names = [tool["name"] for tool in tools_in_category]
+                        result["categories"][category] = {"count": len(tool_names)}
 
-        # Also include remote tools for listing purposes (not auto-loaded elsewhere)
-        try:
-            remote_dir = os.path.join(current_dir, "data", "remote_tools")
-            if os.path.isdir(remote_dir):
-                remote_tools = []
-                for fname in os.listdir(remote_dir):
-                    if not fname.lower().endswith(".json"):
-                        continue
-                    fpath = os.path.join(remote_dir, fname)
-                    try:
-                        tools_in_file = read_json_list(fpath)
-                        if isinstance(tools_in_file, dict):
-                            tools_in_file = list(tools_in_file.values())
-                        if isinstance(tools_in_file, list):
-                            remote_tools.extend(tools_in_file)
-                    except Exception as e:
-                        warning(
-                            f"Warning: Could not read remote tools from {fpath}: {e}"
-                        )
-                if remote_tools:
-                    all_tools.extend(remote_tools)
-                    all_tool_names.update([tool["name"] for tool in remote_tools])
+                except Exception as e:
+                    warning(
+                        f"Warning: Could not read tools from {category} ({file_path}): {e}"
+                    )
                     if mode == "config":
-                        result["categories"]["remote_tools"] = {
-                            "count": len(remote_tools)
-                        }
-        except Exception as e:
-            warning(f"Warning: Failed to scan remote tools directory: {e}")
+                        result["categories"][category] = {"count": 0}
 
         # If mode is 'type', organize by tool types instead
         if mode == "type":
@@ -1202,6 +1210,113 @@ class ToolUniverse:
 
         return result
 
+    def _scan_predefined_files(self):
+        """
+        Scan predefined tool files (original behavior).
+
+        Returns:
+            tuple: (all_tools, all_tool_names) where all_tools is a list of tool configs
+                   and all_tool_names is a set of tool names for deduplication
+        """
+        all_tools = []
+        all_tool_names = set()
+
+        # Read tools from each category file
+        for category, file_path in self.tool_files.items():
+            try:
+                # Read the JSON file for this category
+                tools_in_category = read_json_list(file_path)
+                all_tools.extend(tools_in_category)
+                all_tool_names.update([tool["name"] for tool in tools_in_category])
+            except Exception as e:
+                warning(
+                    f"Warning: Could not read tools from {category} ({file_path}): {e}"
+                )
+
+        # Also include remote tools
+        try:
+            remote_dir = os.path.join(current_dir, "data", "remote_tools")
+            if os.path.isdir(remote_dir):
+                remote_tools = []
+                for fname in os.listdir(remote_dir):
+                    if not fname.lower().endswith(".json"):
+                        continue
+                    fpath = os.path.join(remote_dir, fname)
+                    try:
+                        tools_in_file = read_json_list(fpath)
+                        if isinstance(tools_in_file, dict):
+                            tools_in_file = list(tools_in_file.values())
+                        if isinstance(tools_in_file, list):
+                            remote_tools.extend(tools_in_file)
+                    except Exception as e:
+                        warning(
+                            f"Warning: Could not read remote tools from {fpath}: {e}"
+                        )
+                if remote_tools:
+                    all_tools.extend(remote_tools)
+                    all_tool_names.update([tool["name"] for tool in remote_tools])
+        except Exception as e:
+            warning(f"Warning: Failed to scan remote tools directory: {e}")
+
+        return all_tools, all_tool_names
+
+    def _scan_all_json_files(self):
+        """
+        Recursively scan all JSON files in the data directory and its subdirectories.
+
+        Returns:
+            tuple: (all_tools, all_tool_names) where all_tools is a list of tool configs
+                   and all_tool_names is a set of tool names for deduplication
+        """
+        all_tools = []
+        all_tool_names = set()
+
+        # Get the data directory path
+        data_dir = os.path.join(current_dir, "data")
+
+        if not os.path.exists(data_dir):
+            warning(f"Warning: Data directory not found: {data_dir}")
+            return all_tools, all_tool_names
+
+        # Recursively find all JSON files
+        json_files = []
+        for root, _dirs, files in os.walk(data_dir):
+            for file in files:
+                if file.lower().endswith(".json"):
+                    json_files.append(os.path.join(root, file))
+
+        self.logger.debug(f"Found {len(json_files)} JSON files to scan")
+
+        # Read tools from each JSON file
+        for json_file in json_files:
+            try:
+                tools_in_file = read_json_list(json_file)
+
+                # Handle different data formats
+                if isinstance(tools_in_file, dict):
+                    # Convert dict of tools to list of tools
+                    tools_in_file = list(tools_in_file.values())
+                elif not isinstance(tools_in_file, list):
+                    # Skip files that don't contain tool configurations
+                    continue
+
+                # Add tools to our collection
+                for tool in tools_in_file:
+                    if isinstance(tool, dict) and "name" in tool:
+                        all_tools.append(tool)
+                        all_tool_names.add(tool["name"])
+
+                self.logger.debug(f"Loaded {len(tools_in_file)} tools from {json_file}")
+
+            except Exception as e:
+                warning(f"Warning: Could not read tools from {json_file}: {e}")
+                continue
+
+        self.logger.info(
+            f"Scanned {len(json_files)} JSON files, found {len(all_tools)} tools"
+        )
+        return all_tools, all_tool_names
+
     def refresh_tool_name_desc(
         self,
         enable_full_desc=False,

tooluniverse/scripts/filter_tool_files.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""
+Script to filter tool files by removing tools that don't exist in the current tool universe.
+
+This script:
+1. Gets all valid tool names from ToolUniverse using scan_all=True
+2. Filters tool_relationship_graph_FINAL.json to keep only valid tools
+3. Filters v4_all_tools_final.json to keep only valid tools
+4. Preserves all other data structure and content
+"""
+
+import json
+from pathlib import Path
+
+# Import after modifying sys.path
+from tooluniverse import ToolUniverse
+
+
+def load_json_file(file_path):
+    """Load JSON file and return the data."""
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        print(f"Error loading {file_path}: {e}")
+        return None
+
+
+def save_json_file(file_path, data):
+    """Save data to JSON file."""
+    try:
+        with open(file_path, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+        print(f"Successfully saved filtered data to {file_path}")
+        return True
+    except Exception as e:
+        print(f"Error saving {file_path}: {e}")
+        return False
+
+
+def filter_tool_relationship_graph(data, valid_tool_names):
+    """
+    Filter tool_relationship_graph_FINAL.json to keep only valid tools.
+
+    Args:
+        data: The loaded JSON data
+        valid_tool_names: Set of valid tool names
+
+    Returns:
+        Filtered data
+    """
+    if not isinstance(data, dict):
+        print("Warning: tool_relationship_graph data is not a dict")
+        return data
+
+    filtered_data = {}
+
+    # Handle nodes array
+    if "nodes" in data and isinstance(data["nodes"], list):
+        filtered_nodes = []
+        for node in data["nodes"]:
+            if isinstance(node, dict) and "name" in node:
+                if node["name"] in valid_tool_names:
+                    filtered_nodes.append(node)
+                else:
+                    print(f"Removing node from relationship graph: {node['name']}")
+            else:
+                # Keep non-tool nodes (if any)
+                filtered_nodes.append(node)
+        filtered_data["nodes"] = filtered_nodes
+        print(
+            f"Nodes: {len(data['nodes'])} -> {len(filtered_nodes)} ({len(data['nodes']) - len(filtered_nodes)} removed)"
+        )
+
+    # Handle edges array
+    if "edges" in data and isinstance(data["edges"], list):
+        filtered_edges = []
+        for edge in data["edges"]:
+            if isinstance(edge, dict) and "source" in edge and "target" in edge:
+                # Keep edge if both source and target are valid tools
+                if (
+                    edge["source"] in valid_tool_names
+                    and edge["target"] in valid_tool_names
+                ):
+                    filtered_edges.append(edge)
+                else:
+                    print(
+                        f"Removing edge from relationship graph: {edge.get('source', 'unknown')} -> {edge.get('target', 'unknown')}"
+                    )
+            else:
+                # Keep non-tool edges (if any)
+                filtered_edges.append(edge)
+        filtered_data["edges"] = filtered_edges
+        print(
+            f"Edges: {len(data['edges'])} -> {len(filtered_edges)} ({len(data['edges']) - len(filtered_edges)} removed)"
+        )
+
+    # Keep other fields as-is (like stats, metadata, etc.)
+    for key, value in data.items():
+        if key not in ["nodes", "edges"]:
+            filtered_data[key] = value
+
+    return filtered_data
+
+
+def filter_v4_all_tools(data, valid_tool_names):
+    """
+    Filter v4_all_tools_final.json to keep only valid tools.
+
+    Args:
+        data: The loaded JSON data
+        valid_tool_names: Set of valid tool names
+
+    Returns:
+        Filtered data
+    """
+    if not isinstance(data, list):
+        print("Warning: v4_all_tools data is not a list")
+        return data
+
+    filtered_data = []
+
+    for tool in data:
+        if isinstance(tool, dict) and "name" in tool:
+            if tool["name"] in valid_tool_names:
+                filtered_data.append(tool)
+            else:
+                print(f"Removing tool from v4_all_tools: {tool['name']}")
+        else:
+            # Keep non-tool entries (if any)
+            filtered_data.append(tool)
+
+    return filtered_data
+
+
+def main():
+    """Main function to filter the tool files."""
+    print("Starting tool file filtering process...")
+
+    # Initialize ToolUniverse and get all valid tool names
+    print("Getting all valid tool names from ToolUniverse...")
+    tu = ToolUniverse()
+    all_tool_names = tu.list_built_in_tools(mode="list_name", scan_all=True)
+    valid_tool_names = set(all_tool_names)
+    print(f"Found {len(valid_tool_names)} valid tools")
+
+    # Define file paths
+    project_root = Path(__file__).parent.parent.parent.parent
+    web_dir = project_root / "web"
+
+    relationship_graph_file = web_dir / "tool_relationship_graph_FINAL.json"
+    v4_tools_file = web_dir / "v4_all_tools_final.json"
+
+    # Check if files exist
+    if not relationship_graph_file.exists():
+        print(f"Error: {relationship_graph_file} not found")
+        return
+
+    if not v4_tools_file.exists():
+        print(f"Error: {v4_tools_file} not found")
+        return
+
+    # Process tool_relationship_graph_FINAL.json
+    print(f"\nProcessing {relationship_graph_file.name}...")
+    relationship_data = load_json_file(relationship_graph_file)
+    if relationship_data is not None:
+        len(relationship_data.get("nodes", []))
+        len(relationship_data.get("edges", []))
+        filtered_relationship_data = filter_tool_relationship_graph(
+            relationship_data, valid_tool_names
+        )
+
+        # Save filtered data
+        save_json_file(relationship_graph_file, filtered_relationship_data)
+
+    # Process v4_all_tools_final.json
+    print(f"\nProcessing {v4_tools_file.name}...")
+    v4_data = load_json_file(v4_tools_file)
+    if v4_data is not None:
+        original_count = len(v4_data)
+        filtered_v4_data = filter_v4_all_tools(v4_data, valid_tool_names)
+        filtered_count = len(filtered_v4_data)
+        print(
+            f"V4 tools: {original_count} -> {filtered_count} tools ({original_count - filtered_count} removed)"
+        )
+
+        # Save filtered data
+        save_json_file(v4_tools_file, filtered_v4_data)
+
+    print("\nTool file filtering completed!")
+
+
+if __name__ == "__main__":
+    main()

tooluniverse/test/test_alphafold_tool.py

@@ -1,46 +1,54 @@
 import json
-from typing import Any, Dict, List
 import os
+from typing import Any, Dict, List
 from tooluniverse import ToolUniverse
 
-# Load expected schema fields dynamically from the JSON definition
+# Load all tool schemas from JSON
 schema_path = os.path.join(
     os.path.dirname(__file__), "..", "data", "alphafold_tools.json"
 )
 with open(schema_path) as f:
-    schema = json.load(f)[0]["return_schema"]["properties"]
+    tools_json = json.load(f)
+
+schemas = {tool["name"]: tool["return_schema"] for tool in tools_json}
 
 tooluni = ToolUniverse()
 tooluni.load_tools()
 
-# Test cases: 3 valid, 1 invalid UniProt ID, and 1 missing parameter
 test_queries: List[Dict[str, Any]] = [
+    # Hemoglobin subunit alpha (valid)
     {
-        "name": "alphafold_get_prediction_by_uniprot_id",
-        "arguments": {"uniprot_id": "Q14596"},
+        "name": "alphafold_get_prediction",
+        "arguments": {"qualifier": "P69905"},
     },
+    # Invalid
     {
-        "name": "alphafold_get_prediction_by_uniprot_id",
-        "arguments": {"uniprot_id": "Q9BUR4"},
+        "name": "alphafold_get_prediction",
+        "arguments": {"qualifier": "XXX123"},
     },
+    # Missing param
     {
-        "name": "alphafold_get_prediction_by_uniprot_id",
-        "arguments": {"uniprot_id": "Q8W3K0"},
+        "name": "alphafold_get_prediction",
+        "arguments": {},
     },
+    # Summary: valid
     {
-        "name": "alphafold_get_prediction_by_uniprot_id",
-        "arguments": {"uniprot_id": "XXX123"},
-    },  # invalid
+        "name": "alphafold_get_summary",
+        "arguments": {"qualifier": "P69905"},
+    },
+    # Annotations (valid + invalid type)
     {
-        "name": "alphafold_get_prediction_by_uniprot_id",
-        "arguments": {},
-    },  # missing param
+        "name": "alphafold_get_annotations",
+        "arguments": {"qualifier": "P69905", "type": "MUTAGEN"},
+    },
+    {
+        "name": "alphafold_get_annotations",
+        "arguments": {"qualifier": "P69905", "type": "INVALID"},
+    },
 ]
 
 for idx, query in enumerate(test_queries, 1):
-    uid = query["arguments"].get("uniprot_id")
-    label = f"UniProt ID: {uid}" if uid else "No UniProt ID"
-    print(f"\n[{idx}] Running {query['name']} with {label}")
+    print(f"\n[{idx}] Running {query['name']} with {query['arguments']}")
     result = tooluni.run(query)
 
     # Handle errors
@@ -51,21 +59,50 @@ for idx, query in enumerate(test_queries, 1):
         continue
 
     # Handle success
-    data = result.get("data", [])
+    data = result.get("data")
     if not data:
         print("No data returned.")
         continue
 
-    first = data[0]
-    print("SUCCESS")
-    print(f"   {first.get('uniprotDescription')} ({first.get('uniprotAccession')})")
-    print(f"   Organism: {first.get('organismScientificName')}")
-    print(f"   Avg pLDDT: {first.get('globalMetricValue')}")
-    print(f"   Structure (PDB): {first.get('pdbUrl')}")
+    # Schema validation (check only top-level keys)
+    schema = schemas[query["name"]]
+    expected_keys = schema.get("properties", {}).keys()
+
+    # Handle list vs dict results
+    if isinstance(data, list) and data:
+        record = data[0]
+    elif isinstance(data, dict):
+        record = data
+    else:
+        record = {}
 
-    # Schema validation
-    missing = [k for k in schema.keys() if k not in first]
+    missing = [k for k in expected_keys if k not in record]
     if missing:
         print(f"   INVALID Missing expected fields: {missing}")
     else:
-        print("   All expected schema fields present")
+        print("   SUCCESS All expected schema fields present")
+
+    # Show highlights depending on tool
+    if query["name"] == "alphafold_get_prediction":
+        if "uniprotDescription" in record:
+            print(
+                f"   {record.get('uniprotDescription')} ({record.get('uniprotAccession')})"
+            )
+            print(f"   Organism: {record.get('organismScientificName')}")
+            print(f"   Avg pLDDT: {record.get('globalMetricValue')}")
+
+    elif query["name"] == "alphafold_get_summary":
+        entry = record.get("uniprot_entry", {})
+        structures = record.get("structures", [])
+        print(f"   UniProt AC: {entry.get('ac')}, ID: {entry.get('id')}")
+        print(f"   Sequence length: {entry.get('sequence_length')}")
+        print(f"   Structures returned: {len(structures)}")
+
+    elif query["name"] == "alphafold_get_annotations":
+        annotations = record.get("annotation", [])
+        print(f"   Accession: {record.get('accession')}")
+        print(f"   Total annotations: {len(annotations)}")
+        if annotations:
+            first_ann = annotations[0]
+            print(f"   First annotation type: {first_ann.get('type')}")
+            print(f"   First annotation description: {first_ann.get('description')}")

tooluniverse/test/test_list_built_in_tools.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""
+Basic test for list_built_in_tools including scan_all option.
+Run directly: python tests/test_list_built_in_tools.py
+"""
+
+from tooluniverse import ToolUniverse  # noqa: E402
+
+
+def main():
+    tu = ToolUniverse()
+
+    # Use predefined files (original behavior)
+    tool_names = tu.list_built_in_tools(mode="list_name", scan_all=False)
+    print(f"predefined tool names: {len(tool_names)}")
+
+    # Scan all JSON files
+    all_tool_names = tu.list_built_in_tools(mode="list_name", scan_all=True)
+    print(f"all tool names (scan_all): {len(all_tool_names)}")
+
+    # Get all tool specifications
+    all_tool_specs = tu.list_built_in_tools(mode="list_spec", scan_all=True)
+    print(f"all tool specs (scan_all): {len(all_tool_specs)}")
+
+    # Organize all tools by type
+    type_stats = tu.list_built_in_tools(mode="type", scan_all=True)
+    print(
+        f"type stats -> total_categories: {type_stats['total_categories']}, total_tools: {type_stats['total_tools']}"
+    )
+
+
+if __name__ == "__main__":
+    main()