alita-sdk 0.3.602__py3-none-any.whl → 0.3.609__py3-none-any.whl

alita_sdk/cli/testcases/prompts.py

@@ -0,0 +1,91 @@
+"""
+Prompt building utilities for test execution.
+
+Builds prompts for data generation, test execution, and validation.
+"""
+
+from typing import Dict, Any
+
+
+def build_bulk_data_gen_prompt(parsed_test_cases: list) -> str:
+    """Build consolidated requirements text for bulk test data generation."""
+    requirements = []
+    for idx, tc in enumerate(parsed_test_cases, 1):
+        test_case = tc['data']
+        test_file = tc['file']
+        # Build parts for this test case (do not include separator lines here;
+        # the entire block is wrapped with separators at the top-level)
+        parts = [f"Test Case #{idx}: {test_case['name']}", f"File: {test_file.name}", ""]
+        
+        if test_case.get('test_data_config'):
+            parts.append("Test Data Configuration:")
+            td = test_case['test_data_config']
+            raw_lines = str(td).splitlines()
+            for line in raw_lines:
+                parts.append(f"{line}")
+    
+        if test_case.get('prerequisites'):
+            parts.append(f"\nPre-requisites:\n{test_case['prerequisites']}")
+        
+        requirements.append("\n".join(parts))
+    
+    # If no requirements were collected, return an empty string to avoid
+    # producing a prompt with only separator lines.
+    if not requirements:
+        return ""
+
+    # Use a visible divider between test cases so each entry is clearly separated
+    divider = '-' * 40
+    body = f"\n\n{divider}\n\n".join(requirements)
+    return f"{('='*60)}\n\n{body}\n\n{('='*60)}"
+
+
+def build_single_test_execution_prompt(test_case_info: Dict[str, Any], test_number: int) -> str:
+    """Build execution prompt for a single test case."""
+    test_case = test_case_info['data']
+    test_file = test_case_info['file']
+    
+    parts = [
+        f"\n{'='*80}",
+        f"TEST CASE #{test_number}: {test_case['name']}",
+        f"File: {test_file.name}",
+        f"{'='*80}"
+    ]
+    
+    if test_case['steps']:
+        for step in test_case['steps']:
+            parts.append(f"\nStep {step['number']}: {step['title']}")
+            parts.append(step['instruction'])
+    else:
+        parts.append("\n(No steps defined)")
+    
+    return "\n".join(parts)
+
+
+def build_single_test_validation_prompt(test_case_info: Dict[str, Any], test_number: int, execution_output: str) -> str:
+    """Build validation prompt for a single test case."""
+    test_case = test_case_info['data']
+    
+    parts = [
+        f"\nTest Case #{test_number}: {test_case['name']}"
+    ]
+    
+    if test_case['steps']:
+        for step in test_case['steps']:
+            parts.append(f"  Step {step['number']}: {step['title']}")
+            if step['expectation']:
+                parts.append(f"    Expected: {step['expectation']}")
+    
+    parts.append(f"\n\nActual Execution Results:\n{execution_output}\n")
+    
+    # Escape quotes in test name for valid JSON in prompt
+    escaped_test_name = test_case['name'].replace('"', '\\"')
+    
+    parts.append(f"""\nBased on the execution results above, validate this test case.
+        {{
+          "test_number": {test_number},
+          "test_name": "{escaped_test_name}"
+        }}
+    """)
+    
+    return "\n".join(parts)

alita_sdk/cli/testcases/reporting.py

@@ -0,0 +1,125 @@
+"""
+Test result reporting and summary generation.
+
+Handles generating test reports and displaying summaries.
+"""
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Any
+from rich.console import Console
+from rich.table import Table
+from rich import box
+
+console = Console()
+
+
+def generate_summary_report(test_results: List[Dict[str, Any]]) -> Table:
+    """Generate a summary table for test results.
+    
+    Args:
+        test_results: List of test result dicts
+        
+    Returns:
+        Rich Table with summary statistics
+    """
+    total_tests = len(test_results)
+    passed_tests = sum(1 for r in test_results if r['passed'])
+    failed_tests = total_tests - passed_tests
+    pass_rate = (passed_tests / total_tests * 100) if total_tests > 0 else 0
+    
+    summary_table = Table(box=box.ROUNDED, border_style="cyan")
+    summary_table.add_column("Metric", style="bold")
+    summary_table.add_column("Value", justify="right")
+    
+    summary_table.add_row("Total Tests", str(total_tests))
+    summary_table.add_row("Passed", f"[green]{passed_tests}[/green]")
+    summary_table.add_row("Failed", f"[red]{failed_tests}[/red]")
+    summary_table.add_row("Pass Rate", f"{pass_rate:.1f}%")
+    
+    return summary_table
+
+
+def save_structured_report(
+    test_results: List[Dict[str, Any]],
+    results_dir: str,
+    log_file: Path = None
+) -> Path:
+    """Save structured JSON report of test results.
+    
+    Args:
+        test_results: List of test result dicts
+        results_dir: Directory to save report
+        log_file: Optional path to log file
+        
+    Returns:
+        Path to saved report file
+    """
+    results_path = Path(results_dir)
+    results_path.mkdir(parents=True, exist_ok=True)
+    
+    total_tests = len(test_results)
+    passed_tests = sum(1 for r in test_results if r['passed'])
+    failed_tests = total_tests - passed_tests
+    pass_rate = (passed_tests / total_tests * 100) if total_tests > 0 else 0
+    overall_result = "pass" if failed_tests == 0 else "fail"
+    
+    structured_report = {
+        "test_cases": [
+            {
+                "title": r['title'], 
+                "passed": r['passed'],
+                "steps": r.get('step_results', [])
+            } 
+            for r in test_results
+        ],
+        "overall_result": overall_result,
+        "summary": {
+            "total_tests": total_tests,
+            "passed": passed_tests,
+            "failed": failed_tests,
+            "pass_rate": f"{pass_rate:.1f}%"
+        },
+        "timestamp": datetime.now().isoformat(),
+        "log_file": str(log_file) if log_file else None
+    }
+    
+    summary_file = results_path / "test_execution_summary.json"
+    
+    console.print(f"\n[bold yellow]💾 Saving test execution summary...[/bold yellow]")
+    with open(summary_file, 'w') as f:
+        json.dump(structured_report, f, indent=2)
+    console.print(f"[green]✓ Summary saved to {summary_file}[/green]\n")
+    
+    return summary_file
+
+
+def print_test_execution_summary(
+    test_results: List[Dict[str, Any]],
+    results_dir: str,
+    session_name: str
+) -> None:
+    """Print test execution summary to console.
+    
+    Args:
+        test_results: List of test result dicts
+        results_dir: Directory where results are saved
+        session_name: Session name for finding log file
+    """
+    console.print(f"\n[bold]{'='*60}[/bold]")
+    console.print(f"[bold cyan]📊 Test Execution Summary[/bold cyan]")
+    console.print(f"[bold]{'='*60}[/bold]\n")
+    
+    summary_table = generate_summary_report(test_results)
+    console.print(summary_table)
+    
+    # Show log file location
+    results_path = Path(results_dir)
+    toolkit_name = session_name.replace('test-execution-', '')
+    toolkit_dir = results_path / toolkit_name
+    log_files = sorted(toolkit_dir.glob(f"*{session_name}.txt")) if toolkit_dir.exists() else []
+    
+    console.print(f"\n[bold cyan]📁 Log File[/bold cyan]")
+    if log_files:
+        console.print(f"  [dim]{log_files[0]}[/dim]")

alita_sdk/cli/testcases/setup.py

@@ -0,0 +1,108 @@
+"""
+Agent setup utilities for test execution.
+
+Handles loading and validating test runner, data generator, and validator agents.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any, Tuple
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def load_test_runner_agent(agent_source: str) -> Tuple[Dict[str, Any], str]:
+    """Load test runner agent definition.
+    
+    Args:
+        agent_source: Path to agent definition file
+        
+    Returns:
+        Tuple of (agent_def, agent_name)
+        
+    Raises:
+        FileNotFoundError: If agent file doesn't exist
+    """
+    from ..agent_loader import load_agent_definition
+    
+    agent_source_path = Path(agent_source)
+    if not agent_source_path.exists():
+        raise FileNotFoundError(
+            f"Agent definition not found: {agent_source}. "
+            f"Make sure you are running from the repository root, "
+            f"or pass --agent_source explicitly."
+        )
+    
+    agent_def = load_agent_definition(agent_source)
+    agent_name = agent_def.get('name', agent_source_path.stem)
+    
+    return agent_def, agent_name
+
+
+def load_data_generator_agent(data_generator: str, skip_data_generation: bool) -> Optional[Dict[str, Any]]:
+    """Load data generator agent definition if needed.
+    
+    Args:
+        data_generator: Path to data generator agent file
+        skip_data_generation: Whether data generation is skipped
+        
+    Returns:
+        Agent definition dict or None if skipped/failed
+    """
+    from ..agent_loader import load_agent_definition
+    
+    if skip_data_generation:
+        return None
+    
+    if not data_generator:
+        return None
+    
+    try:
+        data_gen_def = load_agent_definition(data_generator)
+        data_gen_name = data_gen_def.get('name', Path(data_generator).stem)
+        console.print(f"Data Generator Agent: [bold]{data_gen_name}[/bold]\n")
+        return data_gen_def
+    except Exception as e:
+        console.print(f"[yellow]⚠ Warning: Failed to setup data generator: {e}[/yellow]")
+        console.print("[yellow]Continuing with test execution...[/yellow]\n")
+        logger.debug(f"Data generator setup error: {e}", exc_info=True)
+        return None
+
+
+def load_validator_agent(validator: Optional[str]) -> Tuple[Optional[Dict[str, Any]], str, Optional[str]]:
+    """Load validator agent definition.
+    
+    Args:
+        validator: Path to validator agent file (optional)
+        
+    Returns:
+        Tuple of (validator_def, validator_name, validator_path)
+    """
+    from ..agent_loader import load_agent_definition
+    
+    validator_def = None
+    validator_agent_name = "Default Validator"
+    validator_path = validator
+    
+    # Try to load validator from specified path or default location
+    if not validator_path:
+        default_validator = Path.cwd() / '.alita' / 'agents' / 'test-validator.agent.md'
+        if default_validator.exists():
+            validator_path = str(default_validator)
+    
+    if validator_path and Path(validator_path).exists():
+        try:
+            validator_def = load_agent_definition(validator_path)
+            validator_agent_name = validator_def.get('name', Path(validator_path).stem)
+            console.print(f"Validator Agent: [bold]{validator_agent_name}[/bold]")
+            console.print(f"[dim]Using: {validator_path}[/dim]\n")
+        except Exception as e:
+            console.print(f"[yellow]⚠ Warning: Failed to load validator agent: {e}[/yellow]")
+            console.print(f"[yellow]Will use test runner agent for validation[/yellow]\n")
+            logger.debug(f"Validator load error: {e}", exc_info=True)
+    else:
+        console.print(f"[dim]No validator agent specified, using test runner agent for validation[/dim]\n")
+    
+    return validator_def, validator_agent_name, validator_path

alita_sdk/cli/testcases/test_runner.py

@@ -0,0 +1,282 @@
+"""
+Single test case execution utilities.
+
+Handles executing a single test case with the test runner agent.
+"""
+
+import logging
+import uuid
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from rich.console import Console
+
+from langchain_core.runnables import RunnableConfig
+
+from ..callbacks import create_cli_callback
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def execute_single_test_case(
+    tc_info: Dict[str, Any],
+    idx: int,
+    total_tests: int,
+    bulk_gen_chat_history: List[Dict[str, str]],
+    test_cases_path: Path,
+    executor_cache: Dict,
+    client,
+    agent_def: Dict[str, Any],
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> Optional[str]:
+    """Execute a single test case.
+    
+    Args:
+        tc_info: Test case info dict with 'data' and 'file'
+        idx: Test case index (1-based)
+        total_tests: Total number of test cases
+        bulk_gen_chat_history: Chat history from data generation
+        test_cases_path: Path to test cases directory
+        executor_cache: Cache of executors
+        client: API client
+        agent_def: Agent definition
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        Execution output string, or None if execution failed
+    """
+    from .parser import resolve_toolkit_config_path
+    from .prompts import build_single_test_execution_prompt
+    from .utils import extract_toolkit_name
+    from .executor import create_executor_from_cache
+    from ..agent_ui import extract_output_from_result
+    
+    test_case = tc_info['data']
+    test_file = tc_info['file']
+    test_name = test_case['name']
+    
+    # Resolve toolkit config path for this test case
+    toolkit_config_path = resolve_toolkit_config_path(
+        test_case.get('config_path', ''),
+        test_file,
+        test_cases_path
+    )
+    
+    # Extract toolkit name
+    toolkit_name = extract_toolkit_name(test_case.get('config_path', ''))
+    
+    # Use cache key (None if no config)
+    cache_key = toolkit_config_path if toolkit_config_path else '__no_config__'
+    thread_id = f"test_case_{idx}_{uuid.uuid4().hex[:8]}"
+    
+    # Log test case header to master log
+    master_log.print(f"\n\n" + "=" * 80)
+    master_log.print(f"[bold cyan]Test Case {idx}/{total_tests} - {test_name}[/bold cyan]")
+    master_log.print(f"[dim]Toolkit: {toolkit_name}[/dim]")
+    master_log.print(f"[dim]Config: {toolkit_config_path or 'None'}[/dim]")
+    master_log.print("=" * 80 + "\n")
+        
+    # Get or create executor from cache
+    agent_executor, memory, mcp_session_manager = create_executor_from_cache(
+        executor_cache, cache_key, client, agent_def, toolkit_config_path,
+        config, model, temperature, max_tokens, work_dir, setup_executor_func
+    )
+    
+    # Build execution prompt for single test case
+    execution_prompt = build_single_test_execution_prompt(tc_info, idx)
+    master_log.print(f"[dim]Executing with {len(bulk_gen_chat_history)} history messages[/dim]")
+    master_log.print(f"[dim]Executing test case with the prompt {execution_prompt}[/dim]")
+    
+    # Execute test case
+    if not agent_executor:
+        master_log.print(f"[red]✗ No agent executor available[/red]")
+        return None
+    
+    invoke_config = None
+    if verbose:
+        cli_callback = create_cli_callback(verbose=True, debug=debug)
+        invoke_config = RunnableConfig(callbacks=[cli_callback], configurable={"thread_id": thread_id})
+
+    with master_log.status(f"[yellow]Executing test case...[/yellow]", spinner="dots"):
+        exec_result = agent_executor.invoke(
+            {
+                "input": execution_prompt,
+                "chat_history": bulk_gen_chat_history,  # ONLY data gen history, no accumulation
+            },
+            config=invoke_config or {"configurable": {"thread_id": thread_id}},
+        )
+    
+    execution_output = extract_output_from_result(exec_result)
+    
+    master_log.print(f"[green]✓ Test case executed[/green]")
+    master_log.print(f"[dim]{execution_output}[/dim]\n")
+    
+    return execution_output
+
+
+def validate_single_test_case(
+    tc_info: Dict[str, Any],
+    idx: int,
+    execution_output: str,
+    bulk_gen_chat_history: List[Dict[str, str]],
+    validation_executor_cache: Dict,
+    cache_key: str,
+    client,
+    validator_def: Optional[Dict[str, Any]],
+    agent_def: Dict[str, Any],
+    toolkit_config_path: Optional[str],
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> Dict[str, Any]:
+    """Validate a single test case execution.
+    
+    Args:
+        tc_info: Test case info dict
+        idx: Test case index (1-based)
+        execution_output: Output from test execution
+        bulk_gen_chat_history: Chat history including data gen and execution
+        validation_executor_cache: Cache of validation executors
+        cache_key: Cache key for executor
+        client: API client
+        validator_def: Validator agent definition (optional)
+        agent_def: Test runner agent definition (fallback)
+        toolkit_config_path: Path to toolkit config
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        Test result dict with validation results
+    """
+    from .prompts import build_single_test_validation_prompt
+    from .validation import extract_json_from_text, print_validation_diagnostics, create_fallback_result_for_test
+    from .executor import create_executor_from_cache
+    from ..agent_ui import extract_output_from_result
+    
+    test_case = tc_info['data']
+    test_file = tc_info['file']
+    test_name = test_case['name']
+    
+    # Validate test case using validation executor with accumulated history
+    validation_prompt = build_single_test_validation_prompt(tc_info, idx, execution_output)
+    
+    master_log.print(f"[bold yellow]🔍 Validating test case (with execution history)...[/bold yellow]")
+    master_log.print(f"[dim]{validation_prompt}[/dim]\n")
+
+    # Create or retrieve isolated validation executor
+    validation_cache_key = f"{cache_key}_validation"
+    validation_agent_def = validator_def if validator_def else agent_def
+    
+    validation_executor, validation_memory, validation_mcp_session = create_executor_from_cache(
+        validation_executor_cache, validation_cache_key, client, validation_agent_def,
+        toolkit_config_path, config, model, temperature, max_tokens, work_dir, setup_executor_func
+    )
+    
+    if validation_cache_key not in validation_executor_cache:
+        master_log.print(f"[dim]Created new isolated validation executor[/dim]")
+    else:
+        master_log.print(f"[dim]Using cached validation executor[/dim]")
+    
+    # For validation, use a separate thread with accumulated chat history (data gen + execution)
+    validation_thread_id = f"validation_{idx}_{uuid.uuid4().hex[:8]}"
+    
+    if not validation_executor:
+        master_log.print(f"[red]✗ No validation executor available[/red]")
+        return create_fallback_result_for_test(test_case, test_file, 'No validation executor')
+    
+    invoke_config = None
+    if verbose:
+        cli_callback = create_cli_callback(verbose=True, debug=debug)
+        invoke_config = RunnableConfig(callbacks=[cli_callback], configurable={"thread_id": validation_thread_id})
+
+    master_log.print(f"[dim]Executing with {len(bulk_gen_chat_history)} history messages[/dim]")
+    with master_log.status(f"[yellow]Validating test case...[/yellow]", spinner="dots"):
+        validation_result = validation_executor.invoke(
+            {
+                "input": validation_prompt,
+                "chat_history": bulk_gen_chat_history,  # Includes data gen and execution history
+            },
+            config=invoke_config or {"configurable": {"thread_id": validation_thread_id}},
+        )
+    
+    validation_output = extract_output_from_result(validation_result)
+    
+    # Parse validation JSON
+    try:
+        validation_json = extract_json_from_text(validation_output)
+        step_results = validation_json.get('steps', [])
+        
+        # Determine if test passed (all steps must pass)
+        test_passed = all(step.get('passed', False) for step in step_results) if step_results else False
+        
+        if test_passed:
+            master_log.print(f"[bold green]✅ Test PASSED: {test_name}[/bold green]")
+        else:
+            master_log.print(f"[bold red]❌ Test FAILED: {test_name}[/bold red]")
+        
+        # Display individual step results
+        for step_result in step_results:
+            step_num = step_result.get('step_number')
+            step_title = step_result.get('title', '')
+            passed = step_result.get('passed', False)
+            details = step_result.get('details', '')
+            
+            if passed:
+                master_log.print(f"  [green]✓ Step {step_num}: {step_title}[/green]")
+                master_log.print(f"  [dim]{details}[/dim]")
+            else:
+                master_log.print(f"  [red]✗ Step {step_num}: {step_title}[/red]")
+                master_log.print(f"  [dim]{details}[/dim]")
+        
+        master_log.print()
+        
+        return {
+            'title': test_name,
+            'passed': test_passed,
+            'file': test_file.name,
+            'step_results': step_results
+        }
+        
+    except Exception as e:
+        logger.debug(f"Validation parsing failed for {test_name}: {e}", exc_info=True)
+        master_log.print(f"[yellow]⚠ Warning: Could not parse validation results for {test_name}[/yellow]")
+        master_log.print(f"[yellow]Error: {str(e)}[/yellow]")
+        
+        # Enhanced diagnostic output
+        print_validation_diagnostics(validation_output)
+        
+        # Generate fallback result
+        master_log.print(f"\n[yellow]🔄 Generating fallback validation result...[/yellow]")
+        fallback_result = create_fallback_result_for_test(
+            test_case,
+            test_file,
+            f'Validation failed - could not parse validator output: {str(e)}'
+        )
+        master_log.print(f"[dim]Created {len(fallback_result['step_results'])} fallback step results[/dim]\n")
+        
+        return fallback_result

alita_sdk/cli/testcases/utils.py

@@ -0,0 +1,39 @@
+"""
+General utility functions for test execution.
+
+Includes toolkit name extraction and other helper functions.
+"""
+
+from pathlib import Path
+from typing import Optional
+
+
+def extract_toolkit_name(config_path: Optional[str]) -> str:
+    """
+    Extract toolkit name from config path.
+    
+    Args:
+        config_path: Path to toolkit config (e.g., '.alita/tool_configs/github-config.json')
+        
+    Returns:
+        Toolkit name (e.g., 'github') or 'unknown' if path is None/empty
+    """
+    if not config_path:
+        return 'unknown'
+    
+    # Convert to Path
+    path = Path(config_path)
+    
+    # First, try to extract from filename by removing common config suffixes
+    # For paths like '.alita/tool_configs/confluence-config.json' -> 'confluence'
+    stem = path.stem.replace('_config', '').replace('-config', '')
+    if stem and stem.lower() != 'config':
+        return stem
+    
+    # Fallback: use parent directory name if it's not a common directory
+    # For paths like 'toolkits/github/config.yaml' -> 'github'
+    if path.parent.name and path.parent.name not in ['.', 'toolkits', 'tool_configs', 'configs']:
+        return path.parent.name
+    
+    # Last resort
+    return 'unknown'