universal-mcp 0.1.23rc2__py3-none-any.whl → 0.1.24rc3__py3-none-any.whl

universal_mcp/utils/openapi/postprocessor.py

@@ -0,0 +1,275 @@
+import ast
+import re
+
+import litellm
+
+
+def add_hint_tags_to_docstrings(input_path: str, output_path: str):
+    """
+    Reads a Python API client file, inspects each function, and adds appropriate tags to the docstring:
+    - 'readOnlyHint': Tool does not modify its environment (fetching, reading, etc.)
+    - 'destructiveHint': Tool may perform destructive updates
+    - 'openWorldHint': Tool interacts with external entities (3rd party APIs)
+
+    Functions can have multiple tags (e.g., 'readOnlyHint, openWorldHint').
+    Does not alter other tags in the docstring.
+    Writes the modified code to output_path.
+    """
+    with open(input_path, encoding="utf-8") as f:
+        source = f.read()
+    tree = ast.parse(source)
+
+    # Initialize counters
+    total_functions = 0
+    functions_with_http_methods = 0
+    functions_processed_by_llm = 0
+    functions_tagged = 0
+    llm_failures = 0
+
+    class DocstringTagAdder(ast.NodeTransformer):
+        def _find_http_method(self, node):
+            """Find the HTTP method used in the function body."""
+            http_methods = []
+
+            def visit_node(n):
+                if (
+                    isinstance(n, ast.Call)
+                    and isinstance(n.func, ast.Attribute)
+                    and isinstance(n.func.value, ast.Name)
+                    and n.func.value.id == "self"
+                    and n.func.attr in ["_get", "_post", "_put", "_patch", "_delete"]
+                ):
+                    http_methods.append(n.func.attr[1:])
+                for child in ast.iter_child_nodes(n):
+                    visit_node(child)
+
+            visit_node(node)
+            return http_methods[0] if http_methods else None
+
+        def visit_FunctionDef(self, node):
+            nonlocal \
+                total_functions, \
+                functions_with_http_methods, \
+                functions_processed_by_llm, \
+                functions_tagged, \
+                llm_failures
+
+            total_functions += 1
+            print(f"\n[{total_functions}] Processing function: {node.name}")
+
+            http_method = self._find_http_method(node)
+            tag_to_add = None
+
+            if http_method:
+                functions_with_http_methods += 1
+                print(f"  └─ Found HTTP method: {http_method.upper()}")
+
+                # Use simple agent to decide tag
+                print("  └─ Calling LLM to determine tag...")
+                tag_to_add = self._get_tag_suggestion_from_agent(node, http_method)
+
+                if tag_to_add:
+                    functions_processed_by_llm += 1
+                    print(f"  └─ LLM suggested tags: {tag_to_add}")
+                else:
+                    print("  └─ LLM failed or returned invalid response")
+            else:
+                print("  └─ No HTTP method found - skipping")
+
+            if tag_to_add:
+                docstring = ast.get_docstring(node, clean=False)
+                if docstring is not None:
+                    # Look for Tags: section in the docstring
+                    tags_match = re.search(r"Tags:\s*(.+)", docstring, re.DOTALL)
+                    if tags_match:
+                        tags_line = tags_match.group(1).strip()
+                        # Parse existing tags
+                        existing_tags = [tag.strip() for tag in tags_line.split(",")]
+
+                        # Parse new tags to add
+                        new_tags_to_add = [tag.strip() for tag in tag_to_add.split(",")]
+                        tags_to_add = [tag for tag in new_tags_to_add if tag not in existing_tags]
+
+                        if tags_to_add:
+                            # Add the new tags to the existing list
+                            new_tags_line = tags_line.rstrip() + f", {', '.join(tags_to_add)}"
+                            new_docstring = re.sub(r"(Tags:\s*)(.+)", r"\1" + new_tags_line, docstring, flags=re.DOTALL)
+                            # Replace docstring
+                            if isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, ast.Constant):
+                                node.body[0].value.value = new_docstring
+                                functions_tagged += 1
+                                print(f"  └─ ✅ Tags '{', '.join(tags_to_add)}' added successfully")
+                        else:
+                            print(f"  └─ ⚠️  All tags '{tag_to_add}' already exist - skipping")
+                    else:
+                        print("  └─ ⚠️  No 'Tags:' section found in docstring - skipping")
+                else:
+                    print("  └─ ⚠️  No docstring found - skipping")
+            return node
+
+        def _get_tag_suggestion_from_agent(self, node, http_method):
+            """Use a simple agent to decide which tag to add based on function context."""
+
+            function_name = node.name
+            docstring = ast.get_docstring(node, clean=False) or ""
+            parameters = [arg.arg for arg in node.args.args if arg.arg != "self"]
+
+            system_prompt = """You are an expert at analyzing API functions and determining their characteristics.
+
+            Your task is to analyze each function and decide which tags to add:
+            - 'readOnlyHint': Tool does not modify its environment (fetching, reading, etc.)
+            - 'destructiveHint': Tool may perform destructive updates
+            - 'openWorldHint': Tool interacts with external entities (3rd party APIs)
+
+            IMPORTANT:
+            - HTTP method alone is NOT enough to determine the tags. You must analyze the function's actual purpose.
+            - Since these are all API client functions, MOST functions should have 'openWorldHint' (they interact with external APIs).
+            - Only functions that are purely local operations (like reading local files) should NOT have 'openWorldHint'.
+
+            Functions can have multiple tags. For example:
+            - A function that reads from Gmail API: 'readOnlyHint, openWorldHint'
+            - A function that deletes from GitHub API: 'destructiveHint, openWorldHint'
+            - A function that only reads local files: 'readOnlyHint' (no openWorldHint)
+
+            Respond with comma-separated tags (e.g., 'readOnlyHint, openWorldHint') or 'none' if no tags apply."""
+
+            user_prompt = f"""Analyze this API function and decide which tags to add:
+
+Function Name: {function_name}
+HTTP Method: {http_method}
+Parameters: {", ".join(parameters)}
+Docstring: {docstring[:1000]}...
+
+Based on this information, which tags should this function get?
+
+Think through:
+1. What does this function actually do? (from name and docstring)
+2. Does it modify its environment or just read/fetch?
+3. Does it interact with external entities (3rd party APIs)?
+4. Could it be potentially destructive?
+
+GUIDELINES for readOnlyHint (does not modify environment):
+- Functions that only READ or FETCH data
+- Functions that VALIDATE or CHECK things without saving
+- Functions that EXPORT or DOWNLOAD data
+- Functions that perform HEALTH CHECKS or PING operations
+- Functions that REFRESH tokens or sessions
+- Functions that SEARCH or FILTER data
+- Functions that GET information without changing anything
+- Functions that LIST or RETRIEVE data
+
+GUIDELINES for destructiveHint (DESTROYS or DELETES things):
+- Functions that DELETE resources or data
+- Functions that REMOVE or ERASE things
+- Functions that DESTROY or TERMINATE resources
+- Functions that CANCEL or ABORT operations
+- Functions that REVOKE or INVALIDATE things
+
+IMPORTANT:
+- A function should NOT have both readOnlyHint and destructiveHint - they are mutually exclusive.
+- Creating, sending, or updating things is NOT destructive - only deleting/destroying is destructive.
+- Functions that CREATE, SEND, UPDATE, or MODIFY should NOT get destructiveHint.
+
+GUIDELINES for openWorldHint (interacts with external entities):
+- Functions that interact with 3rd party APIs (Gmail, Outlook, Reddit, GitHub, etc.)
+- Functions that make external HTTP requests
+- Functions that connect to external services
+- Functions that interact with cloud services
+- Functions that communicate with external databases
+- Functions that call external webhooks
+- MOST API client functions will have this tag since they interact with external APIs
+
+NOT openWorldHint (local operations):
+- Functions that only read local files
+- Functions that process local data
+- Functions that work with local databases
+- Functions that manipulate local variables
+- Functions that only work with local system resources
+
+Examples:
+- Gmail API read function: 'readOnlyHint, openWorldHint'
+- Gmail API send email: 'openWorldHint' (not destructive, just sending)
+- Gmail API create draft: 'openWorldHint' (not destructive, just creating)
+- GitHub API delete repository: 'destructiveHint, openWorldHint'
+- Local file reader: 'readOnlyHint' (no openWorldHint)
+- Local data processor: 'none' (no tags)
+
+Focus on the FUNCTION'S PURPOSE, not just the HTTP method.
+
+Your answer (comma-separated tags or 'none'):"""
+
+            try:
+                response = litellm.completion(
+                    model="perplexity/sonar-pro",
+                    messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}],
+                    temperature=0.1,
+                    max_tokens=50,
+                )
+
+                suggested_tags = response.choices[0].message.content.strip().lower()
+
+                if suggested_tags == "none":
+                    return None
+
+                # Parse comma-separated tags
+                tag_list = [tag.strip() for tag in suggested_tags.split(",")]
+                valid_tags = []
+
+                for tag in tag_list:
+                    if tag == "readonlyhint":
+                        valid_tags.append("readOnlyHint")
+                    elif tag == "destructivehint":
+                        valid_tags.append("destructiveHint")
+                    elif tag == "openworldhint":
+                        valid_tags.append("openWorldHint")
+
+                if valid_tags:
+                    return ", ".join(valid_tags)
+                else:
+                    # If LLM gives unexpected response, return None (no tag added)
+                    return None
+
+            except Exception as e:
+                nonlocal llm_failures
+                llm_failures += 1
+                print(f"  └─ ❌ LLM failed for function {function_name}: {e}")
+                # If LLM fails, return None (no tag added)
+                return None
+
+    new_tree = DocstringTagAdder().visit(tree)
+    ast.fix_missing_locations(new_tree)
+    new_source = ast.unparse(new_tree)
+
+    # Print summary statistics
+    print(f"\n{'=' * 60}")
+    print("📊 PROCESSING SUMMARY")
+    print(f"{'=' * 60}")
+    print(f"Total functions processed: {total_functions}")
+    print(f"Functions with HTTP methods: {functions_with_http_methods}")
+    print(f"Functions processed by LLM: {functions_processed_by_llm}")
+    print(f"Functions successfully tagged: {functions_tagged}")
+    print(f"LLM failures: {llm_failures}")
+    if functions_with_http_methods > 0:
+        print(
+            f"LLM success rate: {(functions_processed_by_llm / functions_with_http_methods * 100):.1f}% of HTTP functions"
+        )
+    print(f"{'=' * 60}")
+
+    # Format with Black in memory
+    try:
+        import black
+
+        formatted_content = black.format_file_contents(new_source, fast=False, mode=black.FileMode())
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write(formatted_content)
+        print(f"Black formatting applied successfully to: {output_path}")
+    except ImportError:
+        print(f"Black not installed. Skipping formatting for: {output_path}")
+        # Write unformatted version if Black is not available
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write(new_source)
+    except Exception as e:
+        print(f"Black formatting failed for {output_path}: {e}")
+        # Write unformatted version if Black formatting fails
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write(new_source)

universal_mcp/utils/openapi/preprocessor.py

@@ -12,6 +12,8 @@ import typer
 import yaml
 from rich.console import Console
 
+from .filters import load_filter_config, should_process_operation
+
 console = Console()
 
 
@@ -225,7 +227,7 @@ def generate_description_llm(
         if len(param_context_str) > 1000:  # Limit context size
             param_context_str = param_context_str[:1000] + "..."
 
-        current_description = context.get("current_description", None)
+        current_description = context.get("current_description")
         if current_description and isinstance(current_description, str) and current_description.strip():
             user_prompt = f"""The current description for the API parameter named '{param_name}' located '{param_in}' for the '{method.upper()}' operation at path '{path_key}' is:\n'{current_description.strip()}'\n\nTask: Rewrite and enrich this description so it is clear, self-contained, and makes sense to a user. If the description is cut off, incomplete, or awkward, make it complete and natural. Ensure it is concise and under {MAX_DESCRIPTION_LENGTH} characters. Do not include any links, HTML, markdown, or any notes or comments about the character limit. Respond ONLY with the improved single-line description."""
             fallback_text = (
@@ -451,11 +453,12 @@ def simplify_parameter_context(parameter: dict) -> dict:
     return simplified_context
 
 
-def scan_schema_for_status(schema_data: dict):
+def scan_schema_for_status(schema_data: dict, filter_config: dict[str, str | list[str]] | None = None):
     """
     Scans the schema to report the status of descriptions/summaries
     and identify critical issues like missing parameter 'name'/'in'.
     Does NOT modify the schema or call the LLM.
+    Respects filter configuration if provided.
     """
     logger.info("\n--- Scanning Schema for Status ---")
 
@@ -526,6 +529,12 @@ def scan_schema_for_status(schema_data: dict):
                 "trace",
             ]:
                 operation_location_base = f"paths.{path_key}.{method.lower()}"
+
+                # Apply filter configuration
+                if not should_process_operation(path_key, method, filter_config):
+                    logger.debug(f"Skipping operation '{method.upper()} {path_key}' due to filter configuration.")
+                    continue
+
                 if not isinstance(operation_value, dict):
                     logger.warning(f"Operation value for '{operation_location_base}' is not a dictionary. Skipping.")
                     continue
@@ -886,12 +895,20 @@ def process_operation(
 
 
 def process_paths(
-    paths: dict, llm_model: str, enhance_all: bool, summaries_only: bool = False, operation_ids_only: bool = False
+    paths: dict,
+    llm_model: str,
+    enhance_all: bool,
+    summaries_only: bool = False,
+    operation_ids_only: bool = False,
+    filter_config: dict[str, str | list[str]] | None = None,
 ):
     if not isinstance(paths, dict):
         logger.warning("'paths' field is not a dictionary. Skipping path processing.")
         return
 
+    processed_count = 0
+    skipped_count = 0
+
     for path_key, path_value in paths.items():
         if path_key.lower().startswith("x-"):
             logger.debug(f"Skipping processing of path extension '{path_key}'.")
@@ -909,9 +926,17 @@ def process_paths(
                     "patch",
                     "trace",
                 ]:
+                    # Apply filter configuration
+                    if not should_process_operation(path_key, method, filter_config):
+                        logger.debug(f"Skipping operation '{method.upper()} {path_key}' due to filter configuration.")
+                        skipped_count += 1
+                        continue
+
+                    logger.info(f"Processing operation: {method.upper()} {path_key}")
                     process_operation(
                         operation_value, path_key, method, llm_model, enhance_all, summaries_only, operation_ids_only
                     )
+                    processed_count += 1
                 elif method.lower().startswith("x-"):
                     logger.debug(f"Skipping processing of method extension '{method.lower()}' in path '{path_key}'.")
                     continue
@@ -928,6 +953,11 @@ def process_paths(
         elif path_value is not None:
             logger.warning(f"Path value for '{path_key}' is not a dictionary. Skipping processing.")
 
+    if filter_config is not None:
+        logger.info(
+            f"Selective processing complete: {processed_count} operations processed, {skipped_count} operations skipped."
+        )
+
 
 def process_info_section(schema_data: dict, llm_model: str, enhance_all: bool):  # New flag
     info = schema_data.get("info")
@@ -1036,7 +1066,12 @@ def regenerate_duplicate_operation_ids(schema_data: dict, llm_model: str):
 
 
 def preprocess_schema_with_llm(
-    schema_data: dict, llm_model: str, enhance_all: bool, summaries_only: bool = False, operation_ids_only: bool = False
+    schema_data: dict,
+    llm_model: str,
+    enhance_all: bool,
+    summaries_only: bool = False,
+    operation_ids_only: bool = False,
+    filter_config: dict[str, str | list[str]] | None = None,
 ):
     """
     Processes the schema to add/enhance descriptions/summaries using an LLM.
@@ -1045,8 +1080,12 @@ def preprocess_schema_with_llm(
     If operation_ids_only is True, only missing operationIds are generated (never overwritten).
     Assumes basic schema structure validation (info, title) has already passed.
     """
+    filter_info = ""
+    if filter_config is not None:
+        filter_info = f" | Selective processing: {len(filter_config)} path specifications"
+
     logger.info(
-        f"\n--- Starting LLM Generation (enhance_all={enhance_all}, summaries_only={summaries_only}, operation_ids_only={operation_ids_only}) ---"
+        f"\n--- Starting LLM Generation (enhance_all={enhance_all}, summaries_only={summaries_only}, operation_ids_only={operation_ids_only}){filter_info} ---"
     )
 
     # Only process info section if not operation_ids_only
@@ -1054,7 +1093,7 @@ def preprocess_schema_with_llm(
         process_info_section(schema_data, llm_model, enhance_all)
 
     paths = schema_data.get("paths")
-    process_paths(paths, llm_model, enhance_all, summaries_only, operation_ids_only)
+    process_paths(paths, llm_model, enhance_all, summaries_only, operation_ids_only, filter_config)
 
     # After process_paths, regenerate_duplicate_operation_ids(schema_data, llm_model)
 
@@ -1066,10 +1105,24 @@ def run_preprocessing(
     output_path: Path | None = None,
     model: str = "perplexity/sonar",
     debug: bool = False,
+    filter_config_path: str | None = None,
 ):
     set_logging_level("DEBUG" if debug else "INFO")
     console.print("[bold blue]--- Starting OpenAPI Schema Preprocessor ---[/bold blue]")
 
+    # Load filter configuration if provided
+    filter_config = None
+    if filter_config_path:
+        try:
+            filter_config = load_filter_config(filter_config_path)
+            console.print("[bold cyan]Selective Processing Mode Enabled[/bold cyan]")
+            console.print(f"[cyan]Filter configuration loaded from: {filter_config_path}[/cyan]")
+            console.print(f"[cyan]Will process {len(filter_config)} path specifications[/cyan]")
+            console.print()
+        except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
+            console.print(f"[red]Error loading filter configuration: {e}[/red]")
+            raise typer.Exit(1) from e
+
     if schema_path is None:
         path_str = typer.prompt(
             "Please enter the path to the OpenAPI schema file (JSON or YAML)",
@@ -1090,7 +1143,7 @@ def run_preprocessing(
 
     # --- Step 2: Scan and Report Status ---
     try:
-        scan_report = scan_schema_for_status(schema_data)
+        scan_report = scan_schema_for_status(schema_data, filter_config)
         report_scan_results(scan_report)
     except Exception as e:
         console.print(f"[red]An unexpected error occurred during schema scanning: {e}[/red]")
@@ -1224,7 +1277,9 @@ def run_preprocessing(
             f"[blue]Starting LLM generation with Enhance All: {enhance_all}, Summaries Only: {summaries_only}, OperationIds Only: {operation_ids_only}[/blue]"
         )
         try:
-            preprocess_schema_with_llm(schema_data, model, enhance_all, summaries_only, operation_ids_only)
+            preprocess_schema_with_llm(
+                schema_data, model, enhance_all, summaries_only, operation_ids_only, filter_config
+            )
             console.print("[green]LLM generation complete.[/green]")
         except Exception as e:
             console.print(f"[red]Error during LLM generation: {e}[/red]")