alita-sdk 0.3.379__py3-none-any.whl → 0.3.627__py3-none-any.whl

alita_sdk/cli/testcases/test_runner.py

@@ -0,0 +1,282 @@
+"""
+Single test case execution utilities.
+
+Handles executing a single test case with the test runner agent.
+"""
+
+import logging
+import uuid
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from rich.console import Console
+
+from langchain_core.runnables import RunnableConfig
+
+from ..callbacks import create_cli_callback
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def execute_single_test_case(
+    tc_info: Dict[str, Any],
+    idx: int,
+    total_tests: int,
+    bulk_gen_chat_history: List[Dict[str, str]],
+    test_cases_path: Path,
+    executor_cache: Dict,
+    client,
+    agent_def: Dict[str, Any],
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> Optional[str]:
+    """Execute a single test case.
+    
+    Args:
+        tc_info: Test case info dict with 'data' and 'file'
+        idx: Test case index (1-based)
+        total_tests: Total number of test cases
+        bulk_gen_chat_history: Chat history from data generation
+        test_cases_path: Path to test cases directory
+        executor_cache: Cache of executors
+        client: API client
+        agent_def: Agent definition
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        Execution output string, or None if execution failed
+    """
+    from .parser import resolve_toolkit_config_path
+    from .prompts import build_single_test_execution_prompt
+    from .utils import extract_toolkit_name
+    from .executor import create_executor_from_cache
+    from ..agent_ui import extract_output_from_result
+    
+    test_case = tc_info['data']
+    test_file = tc_info['file']
+    test_name = test_case['name']
+    
+    # Resolve toolkit config path for this test case
+    toolkit_config_path = resolve_toolkit_config_path(
+        test_case.get('config_path', ''),
+        test_file,
+        test_cases_path
+    )
+    
+    # Extract toolkit name
+    toolkit_name = extract_toolkit_name(test_case.get('config_path', ''))
+    
+    # Use cache key (None if no config)
+    cache_key = toolkit_config_path if toolkit_config_path else '__no_config__'
+    thread_id = f"test_case_{idx}_{uuid.uuid4().hex[:8]}"
+    
+    # Log test case header to master log
+    master_log.print(f"\n\n" + "=" * 80)
+    master_log.print(f"[bold cyan]Test Case {idx}/{total_tests} - {test_name}[/bold cyan]")
+    master_log.print(f"[dim]Toolkit: {toolkit_name}[/dim]")
+    master_log.print(f"[dim]Config: {toolkit_config_path or 'None'}[/dim]")
+    master_log.print("=" * 80 + "\n")
+        
+    # Get or create executor from cache
+    agent_executor, memory, mcp_session_manager = create_executor_from_cache(
+        executor_cache, cache_key, client, agent_def, toolkit_config_path,
+        config, model, temperature, max_tokens, work_dir, setup_executor_func
+    )
+    
+    # Build execution prompt for single test case
+    execution_prompt = build_single_test_execution_prompt(tc_info, idx)
+    master_log.print(f"[dim]Executing with {len(bulk_gen_chat_history)} history messages[/dim]")
+    master_log.print(f"[dim]Executing test case with the prompt {execution_prompt}[/dim]")
+    
+    # Execute test case
+    if not agent_executor:
+        master_log.print(f"[red]✗ No agent executor available[/red]")
+        return None
+    
+    invoke_config = None
+    if verbose:
+        cli_callback = create_cli_callback(verbose=True, debug=debug)
+        invoke_config = RunnableConfig(callbacks=[cli_callback], configurable={"thread_id": thread_id})
+
+    with master_log.status(f"[yellow]Executing test case...[/yellow]", spinner="dots"):
+        exec_result = agent_executor.invoke(
+            {
+                "input": execution_prompt,
+                "chat_history": bulk_gen_chat_history,  # ONLY data gen history, no accumulation
+            },
+            config=invoke_config or {"configurable": {"thread_id": thread_id}},
+        )
+    
+    execution_output = extract_output_from_result(exec_result)
+    
+    master_log.print(f"[green]✓ Test case executed[/green]")
+    master_log.print(f"[dim]{execution_output}[/dim]\n")
+    
+    return execution_output
+
+
+def validate_single_test_case(
+    tc_info: Dict[str, Any],
+    idx: int,
+    execution_output: str,
+    bulk_gen_chat_history: List[Dict[str, str]],
+    validation_executor_cache: Dict,
+    cache_key: str,
+    client,
+    validator_def: Optional[Dict[str, Any]],
+    agent_def: Dict[str, Any],
+    toolkit_config_path: Optional[str],
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> Dict[str, Any]:
+    """Validate a single test case execution.
+    
+    Args:
+        tc_info: Test case info dict
+        idx: Test case index (1-based)
+        execution_output: Output from test execution
+        bulk_gen_chat_history: Chat history including data gen and execution
+        validation_executor_cache: Cache of validation executors
+        cache_key: Cache key for executor
+        client: API client
+        validator_def: Validator agent definition (optional)
+        agent_def: Test runner agent definition (fallback)
+        toolkit_config_path: Path to toolkit config
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        Test result dict with validation results
+    """
+    from .prompts import build_single_test_validation_prompt
+    from .validation import extract_json_from_text, print_validation_diagnostics, create_fallback_result_for_test
+    from .executor import create_executor_from_cache
+    from ..agent_ui import extract_output_from_result
+    
+    test_case = tc_info['data']
+    test_file = tc_info['file']
+    test_name = test_case['name']
+    
+    # Validate test case using validation executor with accumulated history
+    validation_prompt = build_single_test_validation_prompt(tc_info, idx, execution_output)
+    
+    master_log.print(f"[bold yellow]🔍 Validating test case (with execution history)...[/bold yellow]")
+    master_log.print(f"[dim]{validation_prompt}[/dim]\n")
+
+    # Create or retrieve isolated validation executor
+    validation_cache_key = f"{cache_key}_validation"
+    validation_agent_def = validator_def if validator_def else agent_def
+    
+    validation_executor, validation_memory, validation_mcp_session = create_executor_from_cache(
+        validation_executor_cache, validation_cache_key, client, validation_agent_def,
+        toolkit_config_path, config, model, temperature, max_tokens, work_dir, setup_executor_func
+    )
+    
+    if validation_cache_key not in validation_executor_cache:
+        master_log.print(f"[dim]Created new isolated validation executor[/dim]")
+    else:
+        master_log.print(f"[dim]Using cached validation executor[/dim]")
+    
+    # For validation, use a separate thread with accumulated chat history (data gen + execution)
+    validation_thread_id = f"validation_{idx}_{uuid.uuid4().hex[:8]}"
+    
+    if not validation_executor:
+        master_log.print(f"[red]✗ No validation executor available[/red]")
+        return create_fallback_result_for_test(test_case, test_file, 'No validation executor')
+    
+    invoke_config = None
+    if verbose:
+        cli_callback = create_cli_callback(verbose=True, debug=debug)
+        invoke_config = RunnableConfig(callbacks=[cli_callback], configurable={"thread_id": validation_thread_id})
+
+    master_log.print(f"[dim]Executing with {len(bulk_gen_chat_history)} history messages[/dim]")
+    with master_log.status(f"[yellow]Validating test case...[/yellow]", spinner="dots"):
+        validation_result = validation_executor.invoke(
+            {
+                "input": validation_prompt,
+                "chat_history": bulk_gen_chat_history,  # Includes data gen and execution history
+            },
+            config=invoke_config or {"configurable": {"thread_id": validation_thread_id}},
+        )
+    
+    validation_output = extract_output_from_result(validation_result)
+    
+    # Parse validation JSON
+    try:
+        validation_json = extract_json_from_text(validation_output)
+        step_results = validation_json.get('steps', [])
+        
+        # Determine if test passed (all steps must pass)
+        test_passed = all(step.get('passed', False) for step in step_results) if step_results else False
+        
+        if test_passed:
+            master_log.print(f"[bold green]✅ Test PASSED: {test_name}[/bold green]")
+        else:
+            master_log.print(f"[bold red]❌ Test FAILED: {test_name}[/bold red]")
+        
+        # Display individual step results
+        for step_result in step_results:
+            step_num = step_result.get('step_number')
+            step_title = step_result.get('title', '')
+            passed = step_result.get('passed', False)
+            details = step_result.get('details', '')
+            
+            if passed:
+                master_log.print(f"  [green]✓ Step {step_num}: {step_title}[/green]")
+                master_log.print(f"  [dim]{details}[/dim]")
+            else:
+                master_log.print(f"  [red]✗ Step {step_num}: {step_title}[/red]")
+                master_log.print(f"  [dim]{details}[/dim]")
+        
+        master_log.print()
+        
+        return {
+            'title': test_name,
+            'passed': test_passed,
+            'file': test_file.name,
+            'step_results': step_results
+        }
+        
+    except Exception as e:
+        logger.debug(f"Validation parsing failed for {test_name}: {e}", exc_info=True)
+        master_log.print(f"[yellow]⚠ Warning: Could not parse validation results for {test_name}[/yellow]")
+        master_log.print(f"[yellow]Error: {str(e)}[/yellow]")
+        
+        # Enhanced diagnostic output
+        print_validation_diagnostics(validation_output)
+        
+        # Generate fallback result
+        master_log.print(f"\n[yellow]🔄 Generating fallback validation result...[/yellow]")
+        fallback_result = create_fallback_result_for_test(
+            test_case,
+            test_file,
+            f'Validation failed - could not parse validator output: {str(e)}'
+        )
+        master_log.print(f"[dim]Created {len(fallback_result['step_results'])} fallback step results[/dim]\n")
+        
+        return fallback_result

alita_sdk/cli/testcases/utils.py

@@ -0,0 +1,39 @@
+"""
+General utility functions for test execution.
+
+Includes toolkit name extraction and other helper functions.
+"""
+
+from pathlib import Path
+from typing import Optional
+
+
+def extract_toolkit_name(config_path: Optional[str]) -> str:
+    """
+    Extract toolkit name from config path.
+    
+    Args:
+        config_path: Path to toolkit config (e.g., '.alita/tool_configs/github-config.json')
+        
+    Returns:
+        Toolkit name (e.g., 'github') or 'unknown' if path is None/empty
+    """
+    if not config_path:
+        return 'unknown'
+    
+    # Convert to Path
+    path = Path(config_path)
+    
+    # First, try to extract from filename by removing common config suffixes
+    # For paths like '.alita/tool_configs/confluence-config.json' -> 'confluence'
+    stem = path.stem.replace('_config', '').replace('-config', '')
+    if stem and stem.lower() != 'config':
+        return stem
+    
+    # Fallback: use parent directory name if it's not a common directory
+    # For paths like 'toolkits/github/config.yaml' -> 'github'
+    if path.parent.name and path.parent.name not in ['.', 'toolkits', 'tool_configs', 'configs']:
+        return path.parent.name
+    
+    # Last resort
+    return 'unknown'

alita_sdk/cli/testcases/validation.py

@@ -0,0 +1,90 @@
+"""
+Validation utilities for test execution.
+
+Handles JSON extraction, fallback results, and diagnostics.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Any
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def extract_json_from_text(text: str) -> dict:
+    """Extract JSON object from text using brace counting."""
+    start_idx = text.find('{')
+    if start_idx == -1:
+        raise ValueError("No JSON found in text")
+    
+    brace_count = 0
+    end_idx = -1
+    for i, char in enumerate(text[start_idx:], start=start_idx):
+        if char == '{':
+            brace_count += 1
+        elif char == '}':
+            brace_count -= 1
+            if brace_count == 0:
+                end_idx = i + 1
+                break
+    
+    if end_idx == -1:
+        raise ValueError("Could not find matching closing brace")
+    
+    return json.loads(text[start_idx:end_idx])
+
+
+def create_fallback_result_for_test(test_case: Dict[str, Any], test_file: Path, reason: str = 'Validation failed') -> Dict[str, Any]:
+    """Create a fallback result for a single test case with detailed step information.
+    
+    Args:
+        test_case: Parsed test case data
+        test_file: Path to test case file
+        reason: Reason for fallback
+        
+    Returns:
+        Fallback test result dict with step details
+    """
+    fallback_steps = []
+    for step_info in test_case.get('steps', []):
+        fallback_steps.append({
+            'step_number': step_info['number'],
+            'title': step_info['title'],
+            'passed': False,
+            'details': reason
+        })
+    
+    return {
+        'title': test_case['name'],
+        'passed': False,
+        'file': test_file.name,
+        'step_results': fallback_steps,
+        'validation_error': reason
+    }
+
+
+def print_validation_diagnostics(validation_output: str) -> None:
+    """Print diagnostic information for validation output.
+    
+    Args:
+        validation_output: The validation output to diagnose
+    """
+    console.print(f"\n[bold red]🔍 Diagnostic Information:[/bold red]")
+    console.print(f"[dim]Output length: {len(validation_output)} characters[/dim]")
+    
+    # Check for key JSON elements
+    has_json = '{' in validation_output and '}' in validation_output
+    has_fields = 'test_number' in validation_output and 'steps' in validation_output
+    
+    console.print(f"[dim]Has JSON structure: {has_json}[/dim]")
+    console.print(f"[dim]Has required fields: {has_fields}[/dim]")
+    
+    # Show relevant excerpt
+    if len(validation_output) > 400:
+        console.print(f"\n[red]First 200 chars:[/red] [dim]{validation_output[:200]}[/dim]")
+        console.print(f"[red]Last 200 chars:[/red] [dim]{validation_output[-200:]}[/dim]")
+    else:
+        console.print(f"\n[red]Full output:[/red] [dim]{validation_output}[/dim]")

alita_sdk/cli/testcases/workflow.py

@@ -0,0 +1,196 @@
+"""
+Main workflow orchestration for test case execution.
+
+Coordinates the entire test execution flow from parsing to reporting.
+"""
+
+import logging
+import uuid
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+def parse_all_test_cases(
+    test_case_files_list: List[Path],
+    master_log
+) -> List[Dict[str, Any]]:
+    """Parse all test case files.
+    
+    Args:
+        test_case_files_list: List of test case files to parse
+        master_log: Log capture instance
+        
+    Returns:
+        List of parsed test case dicts with 'file' and 'data' keys
+    """
+    from .parser import parse_test_case
+    
+    parsed_test_cases = []
+    for test_file in test_case_files_list:
+        try:
+            test_case = parse_test_case(str(test_file))
+            parsed_test_cases.append({
+                'file': test_file,
+                'data': test_case
+            })
+        except Exception as e:
+            master_log.print(f"[yellow]⚠ Warning: Failed to parse {test_file.name}: {e}[/yellow]")
+            logger.debug(f"Parse error for {test_file.name}: {e}", exc_info=True)
+    
+    return parsed_test_cases
+
+
+def filter_test_cases_needing_data_gen(
+    parsed_test_cases: List[Dict[str, Any]]
+) -> List[Dict[str, Any]]:
+    """Filter test cases that need data generation.
+    
+    Args:
+        parsed_test_cases: All parsed test cases
+        
+    Returns:
+        Filtered list of test cases that require data generation
+    """
+    return [
+        tc for tc in parsed_test_cases 
+        if tc['data'].get('generate_test_data', True)
+    ]
+
+
+def execute_all_test_cases(
+    parsed_test_cases: List[Dict[str, Any]],
+    bulk_gen_chat_history: List[Dict[str, str]],
+    test_cases_path: Path,
+    agent_def: Dict[str, Any],
+    validator_def: Optional[Dict[str, Any]],
+    client,
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> List[Dict[str, Any]]:
+    """Execute all test cases and return results.
+    
+    Args:
+        parsed_test_cases: List of parsed test cases
+        bulk_gen_chat_history: Chat history from data generation
+        test_cases_path: Path to test cases directory
+        agent_def: Test runner agent definition
+        validator_def: Validator agent definition (optional)
+        client: API client
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        List of test result dicts
+    """
+    from .parser import resolve_toolkit_config_path
+    from .utils import extract_toolkit_name
+    from .executor import cleanup_executor_cache
+    from .test_runner import execute_single_test_case, validate_single_test_case
+    from .validation import create_fallback_result_for_test
+    
+    if not parsed_test_cases:
+        master_log.print("[yellow]No test cases to execute[/yellow]")
+        return []
+    
+    master_log.print(f"\n[bold yellow]📋 Executing test cases sequentially...[/bold yellow]\n")
+    
+    # Show data generation context availability
+    if bulk_gen_chat_history:
+        master_log.print(f"[dim]✓ Data generation history available ({len(bulk_gen_chat_history)} messages) - shared with all test cases[/dim]\n")
+    else:
+        master_log.print(f"[dim]ℹ No data generation history (skipped or disabled)[/dim]\n")
+    
+    # Executor caches
+    executor_cache = {}
+    validation_executor_cache = {}
+    
+    # Execute each test case sequentially
+    test_results = []
+    total_tests = len(parsed_test_cases)
+    
+    for idx, tc_info in enumerate(parsed_test_cases, 1):
+        test_case = tc_info['data']
+        test_file = tc_info['file']
+        test_name = test_case['name']
+        
+        try:
+            # Resolve toolkit config path
+            toolkit_config_path = resolve_toolkit_config_path(
+                test_case.get('config_path', ''),
+                test_file,
+                test_cases_path
+            )
+            
+            # Use cache key
+            cache_key = toolkit_config_path if toolkit_config_path else '__no_config__'
+            
+            # Execute single test case
+            execution_output = execute_single_test_case(
+                tc_info, idx, total_tests, bulk_gen_chat_history, test_cases_path,
+                executor_cache, client, agent_def, config, model, temperature,
+                max_tokens, work_dir, master_log, setup_executor_func,
+                verbose=verbose,
+                debug=debug,
+            )
+            
+            if not execution_output:
+                # Create fallback result for failed execution
+                test_results.append({
+                    'title': test_name,
+                    'passed': False,
+                    'file': test_file.name,
+                    'step_results': []
+                })
+                continue
+            
+            # Append execution to history for validation
+            from .prompts import build_single_test_execution_prompt
+            validation_chat_history = bulk_gen_chat_history + [
+                {"role": "user", "content": build_single_test_execution_prompt(tc_info, idx)},
+                {"role": "assistant", "content": execution_output}
+            ]
+            
+            # Validate test case
+            test_result = validate_single_test_case(
+                tc_info, idx, execution_output, validation_chat_history,
+                validation_executor_cache, cache_key, client, validator_def,
+                agent_def, toolkit_config_path, config, model, temperature,
+                max_tokens, work_dir, master_log, setup_executor_func,
+                verbose=verbose,
+                debug=debug,
+            )
+            
+            test_results.append(test_result)
+            
+        except Exception as e:
+            logger.debug(f"Test execution failed for {test_name}: {e}", exc_info=True)
+            master_log.print(f"[red]✗ Test execution failed: {e}[/red]")
+            
+            # Create fallback result
+            fallback_result = create_fallback_result_for_test(
+                test_case,
+                test_file,
+                f'Test execution failed: {str(e)}'
+            )
+            test_results.append(fallback_result)
+            master_log.print()
+    
+    # Cleanup executor caches
+    cleanup_executor_cache(executor_cache, "executor")
+    cleanup_executor_cache(validation_executor_cache, "validation executor")
+    
+    return test_results