universal-mcp 0.1.23rc2__py3-none-any.whl → 0.1.24rc2__py3-none-any.whl

universal_mcp/utils/openapi/cli.py

@@ -0,0 +1,243 @@
+import re
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+# Setup rich console and logging
+console = Console()
+
+app = typer.Typer(name="codegen")
+
+
+@app.command()
+def generate(
+    schema_path: Path = typer.Option(..., "--schema", "-s"),
+    output_path: Path = typer.Option(
+        None,
+        "--output",
+        "-o",
+        help="Output file path - should match the API name (e.g., 'twitter.py' for Twitter API)",
+    ),
+    class_name: str = typer.Option(
+        None,
+        "--class-name",
+        "-c",
+        help="Class name to use for the API client",
+    ),
+):
+    """Generate API client from OpenAPI schema with optional docstring generation.
+
+    The output filename should match the name of the API in the schema (e.g., 'twitter.py' for Twitter API).
+    This name will be used for the folder in applications/.
+    """
+    # Import here to avoid circular imports
+    from universal_mcp.utils.openapi.api_generator import generate_api_from_schema
+
+    if not schema_path.exists():
+        console.print(f"[red]Error: Schema file {schema_path} does not exist[/red]")
+        raise typer.Exit(1)
+
+    try:
+        app_file_data = generate_api_from_schema(
+            schema_path=schema_path,
+            output_path=output_path,
+            class_name=class_name,
+        )
+        if isinstance(app_file_data, dict) and "code" in app_file_data:
+            console.print("[yellow]No output path specified, printing generated code to console:[/yellow]")
+            console.print(app_file_data["code"])
+        elif isinstance(app_file_data, Path):
+            console.print("[green]API client successfully generated and installed.[/green]")
+            console.print(f"[blue]Application file: {app_file_data}[/blue]")
+        else:
+            # Handle the error case from api_generator if validation fails
+            if isinstance(app_file_data, dict) and "error" in app_file_data:
+                console.print(f"[red]{app_file_data['error']}[/red]")
+                raise typer.Exit(1)
+            else:
+                console.print("[red]Unexpected return value from API generator.[/red]")
+                raise typer.Exit(1)
+
+    except Exception as e:
+        console.print(f"[red]Error generating API client: {e}[/red]")
+        raise typer.Exit(1) from e
+
+
+@app.command()
+def readme(
+    file_path: Path = typer.Argument(..., help="Path to the Python file to process"),
+):
+    """Generate a README.md file for the API client."""
+    from universal_mcp.utils.openapi.readme import generate_readme
+
+    readme_file = generate_readme(file_path)
+    console.print(f"[green]README.md file generated at: {readme_file}[/green]")
+
+
+@app.command()
+def docgen(
+    file_path: Path = typer.Argument(..., help="Path to the Python file to process"),
+    model: str = typer.Option(
+        "perplexity/sonar",
+        "--model",
+        "-m",
+        help="Model to use for generating docstrings",
+    ),
+):
+    """Generate docstrings for Python files using LLMs.
+
+    This command uses litellm with structured output to generate high-quality
+    Google-style docstrings for all functions in the specified Python file.
+    """
+    from universal_mcp.utils.openapi.docgen import process_file
+
+    if not file_path.exists():
+        console.print(f"[red]Error: File not found: {file_path}[/red]")
+        raise typer.Exit(1)
+
+    try:
+        processed = process_file(str(file_path), model)
+        console.print(f"[green]Successfully processed {processed} functions[/green]")
+    except Exception as e:
+        console.print(f"[red]Error: {e}[/red]")
+        raise typer.Exit(1) from e
+
+
+@app.command()
+def init(
+    output_dir: Path | None = typer.Option(
+        None,
+        "--output-dir",
+        "-o",
+        help="Output directory for the project (must exist)",
+    ),
+    app_name: str | None = typer.Option(
+        None,
+        "--app-name",
+        "-a",
+        help="App name (letters, numbers, hyphens, underscores only)",
+    ),
+    integration_type: str | None = typer.Option(
+        None,
+        "--integration-type",
+        "-i",
+        help="Integration type (api_key, oauth, agentr, none)",
+        case_sensitive=False,
+        show_choices=True,
+    ),
+):
+    """Initialize a new MCP project using the cookiecutter template."""
+    from cookiecutter.main import cookiecutter
+
+    NAME_PATTERN = r"^[a-zA-Z0-9_-]+$"
+
+    def validate_pattern(value: str, field_name: str) -> None:
+        if not re.match(NAME_PATTERN, value):
+            console.print(
+                f"[red]❌ Invalid {field_name}; only letters, numbers, hyphens, and underscores allowed.[/red]"
+            )
+            raise typer.Exit(code=1)
+
+    # App name
+    if not app_name:
+        app_name = typer.prompt(
+            "Enter the app name",
+            default="app_name",
+            prompt_suffix=" (e.g., reddit, youtube): ",
+        ).strip()
+    validate_pattern(app_name, "app name")
+    app_name = app_name.lower()
+    if not output_dir:
+        path_str = typer.prompt(
+            "Enter the output directory for the project",
+            default=str(Path.cwd()),
+            prompt_suffix=": ",
+        ).strip()
+        output_dir = Path(path_str)
+
+    if not output_dir.exists():
+        try:
+            output_dir.mkdir(parents=True, exist_ok=True)
+            console.print(f"[green]✅ Created output directory at '{output_dir}'[/green]")
+        except Exception as e:
+            console.print(f"[red]❌ Failed to create output directory '{output_dir}': {e}[/red]")
+            raise typer.Exit(code=1) from e
+    elif not output_dir.is_dir():
+        console.print(f"[red]❌ Output path '{output_dir}' exists but is not a directory.[/red]")
+        raise typer.Exit(code=1)
+
+    # Integration type
+    if not integration_type:
+        integration_type = typer.prompt(
+            "Choose the integration type",
+            default="agentr",
+            prompt_suffix=" (api_key, oauth, agentr, none): ",
+        ).lower()
+    if integration_type not in ("api_key", "oauth", "agentr", "none"):
+        console.print("[red]❌ Integration type must be one of: api_key, oauth, agentr, none[/red]")
+        raise typer.Exit(code=1)
+
+    console.print("[blue]🚀 Generating project using cookiecutter...[/blue]")
+    try:
+        cookiecutter(
+            "https://github.com/AgentrDev/universal-mcp-app-template.git",
+            output_dir=str(output_dir),
+            no_input=True,
+            extra_context={
+                "app_name": app_name,
+                "integration_type": integration_type,
+            },
+        )
+    except Exception as exc:
+        console.print(f"❌ Project generation failed: {exc}")
+        raise typer.Exit(code=1) from exc
+
+    project_dir = output_dir / f"{app_name}"
+    console.print(f"✅ Project created at {project_dir}")
+
+
+@app.command()
+def preprocess(
+    schema_path: Path = typer.Option(None, "--schema", "-s", help="Path to the OpenAPI schema file."),
+    output_path: Path = typer.Option(None, "--output", "-o", help="Path to save the processed schema."),
+):
+    from universal_mcp.utils.openapi.preprocessor import run_preprocessing
+
+    """Preprocess an OpenAPI schema using LLM to fill or enhance descriptions."""
+    run_preprocessing(schema_path, output_path)
+
+
+@app.command()
+def split_api(
+    input_app_file: Path = typer.Argument(..., help="Path to the generated app.py file to split"),
+    output_dir: Path = typer.Option(..., "--output-dir", "-o", help="Directory to save the split files"),
+    package_name: str = typer.Option(
+        None, "--package-name", "-p", help="Package name for absolute imports (e.g., 'hubspot')"
+    ),
+):
+    """Splits a single generated API client file into multiple files based on path groups."""
+    from universal_mcp.utils.openapi.api_splitter import split_generated_app_file
+
+    if not input_app_file.exists() or not input_app_file.is_file():
+        console.print(f"[red]Error: Input file {input_app_file} does not exist or is not a file.[/red]")
+        raise typer.Exit(1)
+
+    if not output_dir.exists():
+        output_dir.mkdir(parents=True, exist_ok=True)
+        console.print(f"[green]Created output directory: {output_dir}[/green]")
+    elif not output_dir.is_dir():
+        console.print(f"[red]Error: Output path {output_dir} is not a directory.[/red]")
+        raise typer.Exit(1)
+
+    try:
+        split_generated_app_file(input_app_file, output_dir, package_name)
+        console.print(f"[green]Successfully split {input_app_file} into {output_dir}[/green]")
+    except Exception as e:
+        console.print(f"[red]Error splitting API client: {e}[/red]")
+
+        raise typer.Exit(1) from e
+
+
+if __name__ == "__main__":
+    app()

universal_mcp/utils/openapi/filters.py

@@ -0,0 +1,114 @@
+"""
+Shared filtering utilities for OpenAPI selective processing.
+
+This module contains common functions used by both the preprocessor
+and API client generator for filtering operations based on JSON configuration.
+"""
+
+import json
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def load_filter_config(config_path: str) -> dict[str, str | list[str]]:
+    """
+    Load the JSON filter configuration file for selective processing.
+
+    Expected format:
+    {
+        "/users/{user-id}/profile": "get",
+        "/users/{user-id}/settings": "all",
+        "/orders/{order-id}": ["get", "put", "delete"]
+    }
+
+    Args:
+        config_path: Path to the JSON configuration file
+
+    Returns:
+        Dictionary mapping paths to methods
+
+    Raises:
+        FileNotFoundError: If config file doesn't exist
+        json.JSONDecodeError: If config file is invalid JSON
+        ValueError: If config format is invalid
+    """
+    if not os.path.exists(config_path):
+        raise FileNotFoundError(f"Filter configuration file not found: {config_path}")
+
+    try:
+        with open(config_path, encoding="utf-8") as f:
+            config = json.load(f)
+    except json.JSONDecodeError as e:
+        raise json.JSONDecodeError(f"Invalid JSON in filter config file {config_path}: {e}") from e
+
+    if not isinstance(config, dict):
+        raise ValueError(f"Filter configuration must be a JSON object/dictionary, got {type(config)}")
+
+    # Validate the configuration format
+    for path, methods in config.items():
+        if not isinstance(path, str):
+            raise ValueError(f"Path keys must be strings, got {type(path)} for key: {path}")
+
+        if isinstance(methods, str):
+            if methods != "all" and methods.lower() not in [
+                "get",
+                "post",
+                "put",
+                "delete",
+                "patch",
+                "head",
+                "options",
+                "trace",
+            ]:
+                raise ValueError(f"Invalid method '{methods}' for path '{path}'. Use 'all' or valid HTTP methods.")
+        elif isinstance(methods, list):
+            for method in methods:
+                if not isinstance(method, str) or method.lower() not in [
+                    "get",
+                    "post",
+                    "put",
+                    "delete",
+                    "patch",
+                    "head",
+                    "options",
+                    "trace",
+                ]:
+                    raise ValueError(f"Invalid method '{method}' for path '{path}'. Use valid HTTP methods.")
+        else:
+            raise ValueError(f"Methods must be string or list of strings for path '{path}', got {type(methods)}")
+
+    logger.info(f"Loaded filter configuration with {len(config)} path specifications")
+    return config
+
+
+def should_process_operation(path: str, method: str, filter_config: dict[str, str | list[str]] | None = None) -> bool:
+    """
+    Check if a specific path+method combination should be processed based on filter config.
+
+    Args:
+        path: The API path (e.g., "/users/{user-id}/profile")
+        method: The HTTP method (e.g., "get")
+        filter_config: Optional filter configuration dict
+
+    Returns:
+        True if the operation should be processed, False otherwise
+    """
+    if filter_config is None:
+        return True  # No filter means process everything
+
+    if path not in filter_config:
+        return False  # Path not in config means skip
+
+    allowed_methods = filter_config[path]
+    method_lower = method.lower()
+
+    if allowed_methods == "all":
+        return True
+    elif isinstance(allowed_methods, str):
+        return method_lower == allowed_methods.lower()
+    elif isinstance(allowed_methods, list):
+        return method_lower in [m.lower() for m in allowed_methods]
+
+    return False

universal_mcp/utils/openapi/openapi.py

@@ -9,6 +9,8 @@ import yaml
 from jsonref import replace_refs
 from pydantic import BaseModel
 
+from .filters import load_filter_config, should_process_operation
+
 
 class Parameters(BaseModel):
     name: str
@@ -150,7 +152,13 @@ def _sanitize_identifier(name: str | None) -> str:
 
     # Initial replacements for common non-alphanumeric characters
     sanitized = (
-        name.replace("-", "_").replace(".", "_").replace("[", "_").replace("]", "").replace("$", "_").replace("/", "_").replace("@", "at")
+        name.replace("-", "_")
+        .replace(".", "_")
+        .replace("[", "_")
+        .replace("]", "")
+        .replace("$", "_")
+        .replace("/", "_")
+        .replace("@", "at")
     )
 
     # Remove leading underscores, but preserve a single underscore if the name (after initial replace)
@@ -1012,18 +1020,28 @@ def load_schema(path: Path):
     return _load_and_resolve_references(path)
 
 
-def generate_api_client(schema, class_name: str | None = None):
+def generate_api_client(schema, class_name: str | None = None, filter_config_path: str | None = None):
     """
     Generate a Python API client class from an OpenAPI schema.
 
     Args:
         schema (dict): The OpenAPI schema as a dictionary.
+        class_name (str | None): Optional class name override.
+        filter_config_path (str | None): Optional path to JSON filter configuration file.
 
     Returns:
         str: A string containing the Python code for the API client class.
     """
+    # Load filter configuration if provided
+    filter_config = None
+    if filter_config_path:
+        filter_config = load_filter_config(filter_config_path)
+        print(f"Loaded filter configuration from {filter_config_path} with {len(filter_config)} path specifications")
+
     methods = []
     method_names = []
+    processed_count = 0
+    skipped_count = 0
 
     # Extract API info for naming and base URL
     info = schema.get("info", {})
@@ -1073,10 +1091,21 @@ def generate_api_client(schema, class_name: str | None = None):
     for path, path_info in schema.get("paths", {}).items():
         for method in path_info:
             if method in ["get", "post", "put", "delete", "patch", "options", "head"]:
+                # Apply filter configuration
+                if not should_process_operation(path, method, filter_config):
+                    print(f"Skipping method generation for '{method.upper()} {path}' due to filter configuration.")
+                    skipped_count += 1
+                    continue
+
                 operation = path_info[method]
+                print(f"Generating method for: {method.upper()} {path}")
                 method_code, func_name = _generate_method_code(path, method, operation)
                 methods.append(method_code)
                 method_names.append(func_name)
+                processed_count += 1
+
+    if filter_config is not None:
+        print(f"Selective generation complete: {processed_count} methods generated, {skipped_count} methods skipped.")
 
     # Generate list_tools method with all the function names
     tools_list = ",\n            ".join([f"self.{name}" for name in method_names])

universal_mcp/utils/openapi/preprocessor.py

@@ -12,6 +12,8 @@ import typer
 import yaml
 from rich.console import Console
 
+from .filters import load_filter_config, should_process_operation
+
 console = Console()
 
 
@@ -451,11 +453,12 @@ def simplify_parameter_context(parameter: dict) -> dict:
     return simplified_context
 
 
-def scan_schema_for_status(schema_data: dict):
+def scan_schema_for_status(schema_data: dict, filter_config: dict[str, str | list[str]] | None = None):
     """
     Scans the schema to report the status of descriptions/summaries
     and identify critical issues like missing parameter 'name'/'in'.
     Does NOT modify the schema or call the LLM.
+    Respects filter configuration if provided.
     """
     logger.info("\n--- Scanning Schema for Status ---")
 
@@ -526,6 +529,12 @@ def scan_schema_for_status(schema_data: dict):
                 "trace",
             ]:
                 operation_location_base = f"paths.{path_key}.{method.lower()}"
+
+                # Apply filter configuration
+                if not should_process_operation(path_key, method, filter_config):
+                    logger.debug(f"Skipping operation '{method.upper()} {path_key}' due to filter configuration.")
+                    continue
+
                 if not isinstance(operation_value, dict):
                     logger.warning(f"Operation value for '{operation_location_base}' is not a dictionary. Skipping.")
                     continue
@@ -886,12 +895,20 @@ def process_operation(
 
 
 def process_paths(
-    paths: dict, llm_model: str, enhance_all: bool, summaries_only: bool = False, operation_ids_only: bool = False
+    paths: dict,
+    llm_model: str,
+    enhance_all: bool,
+    summaries_only: bool = False,
+    operation_ids_only: bool = False,
+    filter_config: dict[str, str | list[str]] | None = None,
 ):
     if not isinstance(paths, dict):
         logger.warning("'paths' field is not a dictionary. Skipping path processing.")
         return
 
+    processed_count = 0
+    skipped_count = 0
+
     for path_key, path_value in paths.items():
         if path_key.lower().startswith("x-"):
             logger.debug(f"Skipping processing of path extension '{path_key}'.")
@@ -909,9 +926,17 @@ def process_paths(
                     "patch",
                     "trace",
                 ]:
+                    # Apply filter configuration
+                    if not should_process_operation(path_key, method, filter_config):
+                        logger.debug(f"Skipping operation '{method.upper()} {path_key}' due to filter configuration.")
+                        skipped_count += 1
+                        continue
+
+                    logger.info(f"Processing operation: {method.upper()} {path_key}")
                     process_operation(
                         operation_value, path_key, method, llm_model, enhance_all, summaries_only, operation_ids_only
                     )
+                    processed_count += 1
                 elif method.lower().startswith("x-"):
                     logger.debug(f"Skipping processing of method extension '{method.lower()}' in path '{path_key}'.")
                     continue
@@ -928,6 +953,11 @@ def process_paths(
         elif path_value is not None:
             logger.warning(f"Path value for '{path_key}' is not a dictionary. Skipping processing.")
 
+    if filter_config is not None:
+        logger.info(
+            f"Selective processing complete: {processed_count} operations processed, {skipped_count} operations skipped."
+        )
+
 
 def process_info_section(schema_data: dict, llm_model: str, enhance_all: bool):  # New flag
     info = schema_data.get("info")
@@ -1036,7 +1066,12 @@ def regenerate_duplicate_operation_ids(schema_data: dict, llm_model: str):
 
 
 def preprocess_schema_with_llm(
-    schema_data: dict, llm_model: str, enhance_all: bool, summaries_only: bool = False, operation_ids_only: bool = False
+    schema_data: dict,
+    llm_model: str,
+    enhance_all: bool,
+    summaries_only: bool = False,
+    operation_ids_only: bool = False,
+    filter_config: dict[str, str | list[str]] | None = None,
 ):
     """
     Processes the schema to add/enhance descriptions/summaries using an LLM.
@@ -1045,8 +1080,12 @@ def preprocess_schema_with_llm(
     If operation_ids_only is True, only missing operationIds are generated (never overwritten).
     Assumes basic schema structure validation (info, title) has already passed.
     """
+    filter_info = ""
+    if filter_config is not None:
+        filter_info = f" | Selective processing: {len(filter_config)} path specifications"
+
     logger.info(
-        f"\n--- Starting LLM Generation (enhance_all={enhance_all}, summaries_only={summaries_only}, operation_ids_only={operation_ids_only}) ---"
+        f"\n--- Starting LLM Generation (enhance_all={enhance_all}, summaries_only={summaries_only}, operation_ids_only={operation_ids_only}){filter_info} ---"
     )
 
     # Only process info section if not operation_ids_only
@@ -1054,7 +1093,7 @@ def preprocess_schema_with_llm(
         process_info_section(schema_data, llm_model, enhance_all)
 
     paths = schema_data.get("paths")
-    process_paths(paths, llm_model, enhance_all, summaries_only, operation_ids_only)
+    process_paths(paths, llm_model, enhance_all, summaries_only, operation_ids_only, filter_config)
 
     # After process_paths, regenerate_duplicate_operation_ids(schema_data, llm_model)
 
@@ -1066,10 +1105,24 @@ def run_preprocessing(
     output_path: Path | None = None,
     model: str = "perplexity/sonar",
     debug: bool = False,
+    filter_config_path: str | None = None,
 ):
     set_logging_level("DEBUG" if debug else "INFO")
     console.print("[bold blue]--- Starting OpenAPI Schema Preprocessor ---[/bold blue]")
 
+    # Load filter configuration if provided
+    filter_config = None
+    if filter_config_path:
+        try:
+            filter_config = load_filter_config(filter_config_path)
+            console.print("[bold cyan]Selective Processing Mode Enabled[/bold cyan]")
+            console.print(f"[cyan]Filter configuration loaded from: {filter_config_path}[/cyan]")
+            console.print(f"[cyan]Will process {len(filter_config)} path specifications[/cyan]")
+            console.print()
+        except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
+            console.print(f"[red]Error loading filter configuration: {e}[/red]")
+            raise typer.Exit(1) from e
+
     if schema_path is None:
         path_str = typer.prompt(
             "Please enter the path to the OpenAPI schema file (JSON or YAML)",
@@ -1090,7 +1143,7 @@ def run_preprocessing(
 
     # --- Step 2: Scan and Report Status ---
     try:
-        scan_report = scan_schema_for_status(schema_data)
+        scan_report = scan_schema_for_status(schema_data, filter_config)
         report_scan_results(scan_report)
     except Exception as e:
         console.print(f"[red]An unexpected error occurred during schema scanning: {e}[/red]")
@@ -1224,7 +1277,9 @@ def run_preprocessing(
             f"[blue]Starting LLM generation with Enhance All: {enhance_all}, Summaries Only: {summaries_only}, OperationIds Only: {operation_ids_only}[/blue]"
         )
         try:
-            preprocess_schema_with_llm(schema_data, model, enhance_all, summaries_only, operation_ids_only)
+            preprocess_schema_with_llm(
+                schema_data, model, enhance_all, summaries_only, operation_ids_only, filter_config
+            )
             console.print("[green]LLM generation complete.[/green]")
         except Exception as e:
             console.print(f"[red]Error during LLM generation: {e}[/red]")