alita-sdk 0.3.379__py3-none-any.whl → 0.3.627__py3-none-any.whl

alita_sdk/cli/mcp_loader.py

@@ -0,0 +1,315 @@
+"""
+MCP (Model Context Protocol) server integration.
+
+Handles loading MCP server configurations and connecting to MCP servers
+using langchain-mcp-adapters to load tools with persistent sessions.
+
+Requires: pip install langchain-mcp-adapters
+Documentation: https://docs.langchain.com/oss/python/langchain/mcp
+Issue Reference: https://github.com/langchain-ai/langchain-mcp-adapters/issues/178
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from rich.console import Console
+
+from .config import substitute_env_vars
+
+console = Console()
+logger = logging.getLogger(__name__)
+
+
+def load_mcp_config(file_path: str) -> Dict[str, Any]:
+    """Load MCP configuration from JSON file (VSCode/Copilot format).
+    
+    Args:
+        file_path: Path to mcp.json configuration file
+        
+    Returns:
+        Dictionary with mcpServers configuration, or empty dict if file doesn't exist
+    """
+    path = Path(file_path)
+    
+    if not path.exists():
+        # Gracefully handle missing file - MCP is optional
+        return {}
+    
+    try:
+        with open(path) as f:
+            content = f.read()
+        
+        # Apply environment variable substitution
+        content = substitute_env_vars(content)
+        config = json.loads(content)
+        
+        # Validate structure
+        if 'mcpServers' not in config:
+            console.print(f"[yellow]Warning: {file_path} missing 'mcpServers' key[/yellow]")
+            return {}
+            
+        return config
+    except json.JSONDecodeError as e:
+        console.print(f"[red]Error parsing {file_path}:[/red] {e}")
+        return {}
+    except Exception as e:
+        console.print(f"[red]Error loading {file_path}:[/red] {e}")
+        return {}
+
+
+def load_mcp_tools(agent_def: Dict[str, Any], mcp_config_path: str) -> List[Dict[str, Any]]:
+    """Load MCP tools from agent definition with tool-level filtering.
+    
+    This function creates MCP server configuration objects that will be processed
+    by the runtime layer using langchain-mcp-adapters to connect to MCP servers
+    and load their tools.
+    
+    The actual connection happens in create_mcp_client() which uses:
+    - langchain_mcp_adapters.client.MultiServerMCPClient for connection
+    - langchain_mcp_adapters.tools.load_mcp_tools for tool loading
+    
+    Args:
+        agent_def: Agent definition dictionary containing mcps list
+        mcp_config_path: Path to mcp.json configuration file (workspace-level)
+        
+    Returns:
+        List of MCP server configurations that will be used by MultiServerMCPClient
+        to connect to servers and load tools with filtering applied.
+    """
+    import fnmatch
+    
+    toolkit_configs = []
+    
+    # Get MCP configuration
+    mcps = agent_def.get('mcps', [])
+    if not mcps:
+        return toolkit_configs
+    
+    # Load mcp.json config file from workspace
+    mcp_config = load_mcp_config(mcp_config_path)
+    mcp_servers = mcp_config.get('mcpServers', {})
+    
+    # Process each MCP entry
+    for mcp_entry in mcps:
+        # Handle both string and object formats
+        if isinstance(mcp_entry, str):
+            server_name = mcp_entry
+            agent_selected_tools = None
+            agent_excluded_tools = None
+        elif isinstance(mcp_entry, dict):
+            server_name = mcp_entry.get('name')
+            agent_selected_tools = mcp_entry.get('selected_tools')
+            agent_excluded_tools = mcp_entry.get('excluded_tools')
+        else:
+            console.print(f"[yellow]Warning: Invalid MCP entry format: {mcp_entry}[/yellow]")
+            continue
+        
+        if not server_name:
+            console.print(f"[yellow]Warning: MCP entry missing 'name': {mcp_entry}[/yellow]")
+            continue
+        
+        # Get server configuration
+        server_config = mcp_servers.get(server_name)
+        if not server_config:
+            console.print(f"[yellow]Warning: MCP server '{server_name}' not found in MCP configuration[/yellow]")
+            continue
+        
+        # Determine tool selection (agent overrides server defaults)
+        server_selected_tools = server_config.get('selected_tools')
+        server_excluded_tools = server_config.get('excluded_tools')
+        
+        # Agent overrides take precedence
+        final_selected_tools = agent_selected_tools if agent_selected_tools is not None else server_selected_tools
+        final_excluded_tools = agent_excluded_tools if agent_excluded_tools is not None else server_excluded_tools
+        
+        # Get connection details
+        server_type = server_config.get('type')  # VSCode format: "stdio" or "streamable_http"
+        server_url = server_config.get('url')
+        server_command = server_config.get('command')
+        server_args = server_config.get('args', [])
+        server_env = server_config.get('env', {})
+        stateful = server_config.get('stateful', False)
+        
+        # Build MCP server config for langchain-mcp-adapters
+        # Support both VSCode format (type: "stdio") and our format (command: "...")
+        if server_type == 'stdio' or server_command:
+            # stdio transport (subprocess)
+            if not server_command:
+                console.print(f"[red]Error: MCP server '{server_name}' has type 'stdio' but no 'command'[/red]")
+                continue
+            
+            console.print(f"[yellow]Note: MCP server '{server_name}' uses command/stdio connection[/yellow]")
+            mcp_server_config = {
+                'transport': 'stdio',
+                'command': server_command,
+                'args': server_args or []
+            }
+        elif server_type == 'streamable_http' or server_url:
+            # HTTP-based transport
+            if not server_url:
+                console.print(f"[red]Error: MCP server '{server_name}' has type 'streamable_http' but no 'url'[/red]")
+                continue
+            
+            mcp_server_config = {
+                'transport': 'streamable_http',
+                'url': server_url
+            }
+        else:
+            console.print(f"[red]Error: MCP server '{server_name}' has neither 'type'/'url' nor 'command'[/red]")
+            continue
+        
+        # Add environment variables if specified
+        if server_env:
+            mcp_server_config['env'] = server_env
+        
+        # Wrap in toolkit config format
+        toolkit_config = {
+            'toolkit_type': 'mcp',
+            'name': server_name,
+            'mcp_server_config': mcp_server_config,
+        }
+        
+        # Add tool filtering if specified
+        if final_selected_tools is not None:
+            toolkit_config['selected_tools'] = final_selected_tools
+        if final_excluded_tools is not None:
+            toolkit_config['excluded_tools'] = final_excluded_tools
+        
+        toolkit_configs.append(toolkit_config)
+        
+        # Display loaded tools info
+        if final_selected_tools:
+            tools_display = ', '.join(final_selected_tools)
+            console.print(f"✓ Loaded MCP server: [cyan]{server_name}[/cyan] (selected: {tools_display})")
+        elif final_excluded_tools:
+            console.print(f"✓ Loaded MCP server: [cyan]{server_name}[/cyan] (excluded: {', '.join(final_excluded_tools)})")
+        else:
+            console.print(f"✓ Loaded MCP server: [cyan]{server_name}[/cyan] (all tools)")
+    
+    return toolkit_configs
+
+
+class MCPSessionManager:
+    """Manages persistent MCP sessions for stateful tools like Playwright.
+    
+    Holds MCP client and session context managers alive for the entire agent lifetime.
+    This is critical for stateful MCP servers that maintain state across tool calls
+    (e.g., Playwright browser sessions).
+    
+    See: https://github.com/langchain-ai/langchain-mcp-adapters/issues/178
+    """
+    def __init__(self):
+        self.client = None
+        self.sessions = {}
+        self._session_contexts = {}
+    
+    def __del__(self):
+        """Cleanup sessions when manager is garbage collected."""
+        # Best effort cleanup - sessions will be closed when process exits anyway
+        pass
+
+
+async def load_mcp_tools_async(mcp_toolkit_configs: List[Dict[str, Any]]):
+    """
+    Load actual MCP tools from servers with persistent sessions.
+    
+    IMPORTANT: This function returns an MCPSessionManager that MUST be kept alive
+    for the entire agent session to maintain stateful MCP server state (e.g., browser).
+    
+    Args:
+        mcp_toolkit_configs: List of MCP toolkit configurations with transport details
+        
+    Returns:
+        Tuple of (session_manager, list_of_tools) where:
+        - session_manager: MCPSessionManager that holds client and persistent sessions
+        - list_of_tools: LangChain async tools from MCP that use persistent sessions
+    """
+    if not mcp_toolkit_configs:
+        return None, []
+    
+    try:
+        from langchain_mcp_adapters.client import MultiServerMCPClient
+        from langchain_mcp_adapters.tools import load_mcp_tools
+    except ImportError:
+        console.print("[yellow]Warning: langchain-mcp-adapters not installed. MCP tools will not be available.[/yellow]")
+        console.print("[yellow]Install with: pip install langchain-mcp-adapters[/yellow]")
+        return None, []
+    
+    # Build server configs for MultiServerMCPClient
+    server_configs = {}
+    for config in mcp_toolkit_configs:
+        server_name = config.get('name', 'unknown')
+        mcp_server_config = config.get('mcp_server_config', {})
+        transport = mcp_server_config.get('transport', 'stdio')
+        
+        if transport == 'stdio':
+            command = mcp_server_config.get('command')
+            args = mcp_server_config.get('args', [])
+            env = mcp_server_config.get('env')
+            
+            # Ensure command is a string
+            if isinstance(command, list):
+                command = command[0] if command else None
+            
+            # Build config dict - only include env if it's set
+            config_dict = {
+                'transport': 'stdio',
+                'command': command,
+                'args': args
+            }
+            if env:
+                config_dict['env'] = env
+            
+            server_configs[server_name] = config_dict
+            console.print(f"[dim]MCP server {server_name}: {command} {' '.join(args)}[/dim]")
+            
+        elif transport == 'streamable_http':
+            server_configs[server_name] = {
+                'transport': 'streamable_http',
+                'url': mcp_server_config.get('url'),
+                'headers': mcp_server_config.get('headers')
+            }
+    
+    if not server_configs:
+        return None, []
+    
+    try:
+        # Create session manager to hold everything alive
+        manager = MCPSessionManager()
+        manager.client = MultiServerMCPClient(server_configs)
+        
+        # Create PERSISTENT sessions for each server
+        # This keeps the MCP server subprocess and state alive for stateful tools
+        all_tools = []
+        
+        for server_name in server_configs.keys():
+            # Enter the session context and keep it alive for entire agent lifetime
+            session_context = manager.client.session(server_name)
+            session = await session_context.__aenter__()
+            
+            # Store both for cleanup and reference
+            manager.sessions[server_name] = session
+            manager._session_contexts[server_name] = session_context
+            
+            # Load tools using the persistent session
+            tools = await load_mcp_tools(
+                session,
+                connection=manager.client.connections[server_name],
+                server_name=server_name
+            )
+            all_tools.extend(tools)
+            
+            console.print(f"[dim]Created persistent session for {server_name}: {len(tools)} tools[/dim]")
+        
+        console.print(f"✓ Loaded {len(all_tools)} MCP tools total from {len(mcp_toolkit_configs)} servers with persistent sessions")
+        
+        return manager, all_tools
+            
+    except ImportError:
+        # Already handled above
+        raise
+    except Exception as e:
+        logger.exception(f"Failed to create MCP client and sessions")
+        console.print(f"[red]Error: Failed to load MCP tools: {e}[/red]")
+        return None, []

alita_sdk/cli/testcases/__init__.py

@@ -0,0 +1,94 @@
+"""
+Test execution utilities for Alita CLI.
+
+This package provides utilities for executing test cases, including:
+- Test case parsing
+- Prompt building
+- Result validation
+- Execution logging
+- Executor cache management
+- Agent setup and loading
+- Test case discovery
+- Test data generation
+- Test execution and validation
+- Result reporting
+"""
+
+from .parser import parse_test_case, resolve_toolkit_config_path
+from .prompts import (
+    build_bulk_data_gen_prompt,
+    build_single_test_execution_prompt,
+    build_single_test_validation_prompt
+)
+from .validation import (
+    extract_json_from_text,
+    create_fallback_result_for_test,
+    print_validation_diagnostics
+)
+from .logger import TestLogCapture
+from .executor import create_executor_from_cache, cleanup_executor_cache
+from .utils import extract_toolkit_name
+from .setup import (
+    load_test_runner_agent,
+    load_data_generator_agent,
+    load_validator_agent
+)
+from .discovery import (
+    discover_test_case_files,
+    validate_test_case_files,
+    print_test_execution_header
+)
+from .data_generation import execute_bulk_data_generation
+from .test_runner import execute_single_test_case, validate_single_test_case
+from .reporting import (
+    generate_summary_report,
+    save_structured_report,
+    print_test_execution_summary
+)
+from .workflow import (
+    parse_all_test_cases,
+    filter_test_cases_needing_data_gen,
+    execute_all_test_cases
+)
+
+__all__ = [
+    # Parser
+    'parse_test_case',
+    'resolve_toolkit_config_path',
+    # Prompts
+    'build_bulk_data_gen_prompt',
+    'build_single_test_execution_prompt',
+    'build_single_test_validation_prompt',
+    # Validation
+    'extract_json_from_text',
+    'create_fallback_result_for_test',
+    'print_validation_diagnostics',
+    # Logger
+    'TestLogCapture',
+    # Executor
+    'create_executor_from_cache',
+    'cleanup_executor_cache',
+    # Utils
+    'extract_toolkit_name',
+    # Setup
+    'load_test_runner_agent',
+    'load_data_generator_agent',
+    'load_validator_agent',
+    # Discovery
+    'discover_test_case_files',
+    'validate_test_case_files',
+    'print_test_execution_header',
+    # Data Generation
+    'execute_bulk_data_generation',
+    # Test Runner
+    'execute_single_test_case',
+    'validate_single_test_case',
+    # Reporting
+    'generate_summary_report',
+    'save_structured_report',
+    'print_test_execution_summary',
+    # Workflow
+    'parse_all_test_cases',
+    'filter_test_cases_needing_data_gen',
+    'execute_all_test_cases',
+]

alita_sdk/cli/testcases/data_generation.py

@@ -0,0 +1,119 @@
+"""
+Bulk test data generation utilities.
+
+Handles executing the data generator agent to provision test data.
+"""
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from langgraph.checkpoint.sqlite import SqliteSaver
+
+from langchain_core.runnables import RunnableConfig
+
+from ..callbacks import create_cli_callback
+
+logger = logging.getLogger(__name__)
+
+
+def execute_bulk_data_generation(
+    data_gen_def: Dict[str, Any],
+    test_cases_needing_data_gen: List[Dict[str, Any]],
+    parsed_test_cases: List[Dict[str, Any]],
+    test_cases_path: Path,
+    client,
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> List[Dict[str, str]]:
+    """Execute bulk test data generation.
+    
+    Args:
+        data_gen_def: Data generator agent definition
+        test_cases_needing_data_gen: Test cases requiring data generation
+        parsed_test_cases: All parsed test cases
+        test_cases_path: Path to test cases directory
+        client: API client
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        Chat history list from data generation
+    """
+    from .parser import resolve_toolkit_config_path
+    from .prompts import build_bulk_data_gen_prompt
+    from ..agent_ui import extract_output_from_result
+    
+    master_log.print(f"\n[bold yellow]🔧 Bulk Test Data Generation[/bold yellow]")
+    master_log.print(f"Generating test data for {len(test_cases_needing_data_gen)} test cases...\n")
+    master_log.print(f"[dim]Skipping {len(parsed_test_cases) - len(test_cases_needing_data_gen)} test cases with generateTestData: false[/dim]\n")
+    
+    bulk_data_gen_prompt = build_bulk_data_gen_prompt(test_cases_needing_data_gen)
+    master_log.print(f"Executing test data generation prompt \n{bulk_data_gen_prompt}\n")
+
+    try:
+        # Setup data generator agent
+        bulk_memory = SqliteSaver(sqlite3.connect(":memory:", check_same_thread=False))
+        
+        # Use first test case's config or empty tuple
+        first_config_path = None
+        if parsed_test_cases:
+            first_tc = parsed_test_cases[0]
+            first_config_path = resolve_toolkit_config_path(
+                first_tc['data'].get('config_path', ''),
+                first_tc['file'],
+                test_cases_path
+            )
+        
+        data_gen_config_tuple = (first_config_path,) if first_config_path else ()
+        data_gen_executor, _, _, _, _, _, _ = setup_executor_func(
+            client, data_gen_def, data_gen_config_tuple, config,
+            model, temperature, max_tokens, bulk_memory, work_dir
+        )
+        
+        if data_gen_executor:
+            master_log.print("\n" + "=" * 80)
+            master_log.print("[bold yellow]📋 BULK DATA GENERATION[/bold yellow]")
+            master_log.print("=" * 80 + "\n")
+            invoke_config = None
+            if verbose:
+                cli_callback = create_cli_callback(verbose=True, debug=debug)
+                invoke_config = RunnableConfig(callbacks=[cli_callback])
+            with master_log.status("[yellow]Generating test data for all test cases...[/yellow]", spinner="dots"):
+                bulk_gen_result = data_gen_executor.invoke(
+                    {
+                        "input": bulk_data_gen_prompt,
+                        "chat_history": [],
+                    },
+                    config=invoke_config,
+                )
+            bulk_gen_output = extract_output_from_result(bulk_gen_result)
+            master_log.print(f"[green]✓ Bulk test data generation completed[/green]")
+            master_log.print(f"\n{bulk_gen_output}\n")
+            
+            # Store chat history from data generation to pass to test executors
+            return [
+                {"role": "user", "content": bulk_data_gen_prompt},
+                {"role": "assistant", "content": bulk_gen_output}
+            ]
+        else:
+            master_log.print(f"[yellow]⚠ Warning: Data generator has no executor[/yellow]\n")
+            return []
+            
+    except Exception as e:
+        master_log.print(f"[yellow]⚠ Warning: Bulk data generation failed: {e}[/yellow]")
+        master_log.print("[yellow]Continuing with test execution...[/yellow]\n")
+        logger.debug(f"Bulk data generation error: {e}", exc_info=True)
+        return []

alita_sdk/cli/testcases/discovery.py

@@ -0,0 +1,96 @@
+"""
+Test case discovery and filtering utilities.
+
+Handles finding test case files and filtering based on user selections.
+"""
+
+import logging
+from pathlib import Path
+from typing import List, Tuple
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def discover_test_case_files(
+    test_cases_dir: str,
+    test_case_files: Tuple[str, ...]
+) -> List[Path]:
+    """Discover and filter test case files based on user selection.
+    
+    Args:
+        test_cases_dir: Directory containing test case files
+        test_case_files: Specific test case files to execute (empty = all)
+        
+    Returns:
+        List of test case file paths
+    """
+    test_cases_path = Path(test_cases_dir)
+    
+    if test_case_files:
+        # User specified specific test case files
+        test_case_files_set = set(test_case_files)
+        all_test_cases = sorted(test_cases_path.rglob('TC-*.md'))
+        test_case_files_list = [
+            tc for tc in all_test_cases 
+            if tc.name in test_case_files_set
+        ]
+        
+        # Check if all specified files were found
+        found_names = {tc.name for tc in test_case_files_list}
+        not_found = test_case_files_set - found_names
+        if not_found:
+            console.print(f"[yellow]⚠ Warning: Test case files not found: {', '.join(not_found)}[/yellow]")
+    else:
+        # Execute all test cases
+        test_case_files_list = sorted(test_cases_path.rglob('TC-*.md'))
+    
+    return test_case_files_list
+
+
+def validate_test_case_files(
+    test_case_files_list: List[Path],
+    test_cases_dir: str,
+    test_case_files: Tuple[str, ...]
+) -> bool:
+    """Validate that test case files were found.
+    
+    Args:
+        test_case_files_list: List of discovered test case files
+        test_cases_dir: Directory that was searched
+        test_case_files: Specific files that were requested
+        
+    Returns:
+        True if files were found, False otherwise (prints warning)
+    """
+    if not test_case_files_list:
+        if test_case_files:
+            console.print(f"[yellow]No matching test case files found in {test_cases_dir}[/yellow]")
+        else:
+            console.print(f"[yellow]No test case files found in {test_cases_dir}[/yellow]")
+        return False
+    
+    return True
+
+
+def print_test_execution_header(
+    agent_name: str,
+    test_case_files_list: List[Path],
+    test_case_files: Tuple[str, ...],
+    results_dir: str
+) -> None:
+    """Print test execution header information.
+    
+    Args:
+        agent_name: Name of the test runner agent
+        test_case_files_list: List of test cases to execute
+        test_case_files: Specific files requested (if any)
+        results_dir: Directory where results will be saved
+    """
+    console.print(f"\n[bold cyan]🧪 Test Execution Started[/bold cyan]")
+    console.print(f"Agent: [bold]{agent_name}[/bold]")
+    console.print(f"Test Cases: {len(test_case_files_list)}")
+    if test_case_files:
+        console.print(f"Selected: [cyan]{', '.join(test_case_files)}[/cyan]")
+    console.print(f"Results Directory: {results_dir}\n")