alita-sdk 0.3.462__py3-none-any.whl → 0.3.627__py3-none-any.whl

alita_sdk/cli/testcases/__init__.py

@@ -0,0 +1,94 @@
+"""
+Test execution utilities for Alita CLI.
+
+This package provides utilities for executing test cases, including:
+- Test case parsing
+- Prompt building
+- Result validation
+- Execution logging
+- Executor cache management
+- Agent setup and loading
+- Test case discovery
+- Test data generation
+- Test execution and validation
+- Result reporting
+"""
+
+from .parser import parse_test_case, resolve_toolkit_config_path
+from .prompts import (
+    build_bulk_data_gen_prompt,
+    build_single_test_execution_prompt,
+    build_single_test_validation_prompt
+)
+from .validation import (
+    extract_json_from_text,
+    create_fallback_result_for_test,
+    print_validation_diagnostics
+)
+from .logger import TestLogCapture
+from .executor import create_executor_from_cache, cleanup_executor_cache
+from .utils import extract_toolkit_name
+from .setup import (
+    load_test_runner_agent,
+    load_data_generator_agent,
+    load_validator_agent
+)
+from .discovery import (
+    discover_test_case_files,
+    validate_test_case_files,
+    print_test_execution_header
+)
+from .data_generation import execute_bulk_data_generation
+from .test_runner import execute_single_test_case, validate_single_test_case
+from .reporting import (
+    generate_summary_report,
+    save_structured_report,
+    print_test_execution_summary
+)
+from .workflow import (
+    parse_all_test_cases,
+    filter_test_cases_needing_data_gen,
+    execute_all_test_cases
+)
+
+__all__ = [
+    # Parser
+    'parse_test_case',
+    'resolve_toolkit_config_path',
+    # Prompts
+    'build_bulk_data_gen_prompt',
+    'build_single_test_execution_prompt',
+    'build_single_test_validation_prompt',
+    # Validation
+    'extract_json_from_text',
+    'create_fallback_result_for_test',
+    'print_validation_diagnostics',
+    # Logger
+    'TestLogCapture',
+    # Executor
+    'create_executor_from_cache',
+    'cleanup_executor_cache',
+    # Utils
+    'extract_toolkit_name',
+    # Setup
+    'load_test_runner_agent',
+    'load_data_generator_agent',
+    'load_validator_agent',
+    # Discovery
+    'discover_test_case_files',
+    'validate_test_case_files',
+    'print_test_execution_header',
+    # Data Generation
+    'execute_bulk_data_generation',
+    # Test Runner
+    'execute_single_test_case',
+    'validate_single_test_case',
+    # Reporting
+    'generate_summary_report',
+    'save_structured_report',
+    'print_test_execution_summary',
+    # Workflow
+    'parse_all_test_cases',
+    'filter_test_cases_needing_data_gen',
+    'execute_all_test_cases',
+]

alita_sdk/cli/testcases/data_generation.py

@@ -0,0 +1,119 @@
+"""
+Bulk test data generation utilities.
+
+Handles executing the data generator agent to provision test data.
+"""
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from langgraph.checkpoint.sqlite import SqliteSaver
+
+from langchain_core.runnables import RunnableConfig
+
+from ..callbacks import create_cli_callback
+
+logger = logging.getLogger(__name__)
+
+
+def execute_bulk_data_generation(
+    data_gen_def: Dict[str, Any],
+    test_cases_needing_data_gen: List[Dict[str, Any]],
+    parsed_test_cases: List[Dict[str, Any]],
+    test_cases_path: Path,
+    client,
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> List[Dict[str, str]]:
+    """Execute bulk test data generation.
+    
+    Args:
+        data_gen_def: Data generator agent definition
+        test_cases_needing_data_gen: Test cases requiring data generation
+        parsed_test_cases: All parsed test cases
+        test_cases_path: Path to test cases directory
+        client: API client
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        Chat history list from data generation
+    """
+    from .parser import resolve_toolkit_config_path
+    from .prompts import build_bulk_data_gen_prompt
+    from ..agent_ui import extract_output_from_result
+    
+    master_log.print(f"\n[bold yellow]🔧 Bulk Test Data Generation[/bold yellow]")
+    master_log.print(f"Generating test data for {len(test_cases_needing_data_gen)} test cases...\n")
+    master_log.print(f"[dim]Skipping {len(parsed_test_cases) - len(test_cases_needing_data_gen)} test cases with generateTestData: false[/dim]\n")
+    
+    bulk_data_gen_prompt = build_bulk_data_gen_prompt(test_cases_needing_data_gen)
+    master_log.print(f"Executing test data generation prompt \n{bulk_data_gen_prompt}\n")
+
+    try:
+        # Setup data generator agent
+        bulk_memory = SqliteSaver(sqlite3.connect(":memory:", check_same_thread=False))
+        
+        # Use first test case's config or empty tuple
+        first_config_path = None
+        if parsed_test_cases:
+            first_tc = parsed_test_cases[0]
+            first_config_path = resolve_toolkit_config_path(
+                first_tc['data'].get('config_path', ''),
+                first_tc['file'],
+                test_cases_path
+            )
+        
+        data_gen_config_tuple = (first_config_path,) if first_config_path else ()
+        data_gen_executor, _, _, _, _, _, _ = setup_executor_func(
+            client, data_gen_def, data_gen_config_tuple, config,
+            model, temperature, max_tokens, bulk_memory, work_dir
+        )
+        
+        if data_gen_executor:
+            master_log.print("\n" + "=" * 80)
+            master_log.print("[bold yellow]📋 BULK DATA GENERATION[/bold yellow]")
+            master_log.print("=" * 80 + "\n")
+            invoke_config = None
+            if verbose:
+                cli_callback = create_cli_callback(verbose=True, debug=debug)
+                invoke_config = RunnableConfig(callbacks=[cli_callback])
+            with master_log.status("[yellow]Generating test data for all test cases...[/yellow]", spinner="dots"):
+                bulk_gen_result = data_gen_executor.invoke(
+                    {
+                        "input": bulk_data_gen_prompt,
+                        "chat_history": [],
+                    },
+                    config=invoke_config,
+                )
+            bulk_gen_output = extract_output_from_result(bulk_gen_result)
+            master_log.print(f"[green]✓ Bulk test data generation completed[/green]")
+            master_log.print(f"\n{bulk_gen_output}\n")
+            
+            # Store chat history from data generation to pass to test executors
+            return [
+                {"role": "user", "content": bulk_data_gen_prompt},
+                {"role": "assistant", "content": bulk_gen_output}
+            ]
+        else:
+            master_log.print(f"[yellow]⚠ Warning: Data generator has no executor[/yellow]\n")
+            return []
+            
+    except Exception as e:
+        master_log.print(f"[yellow]⚠ Warning: Bulk data generation failed: {e}[/yellow]")
+        master_log.print("[yellow]Continuing with test execution...[/yellow]\n")
+        logger.debug(f"Bulk data generation error: {e}", exc_info=True)
+        return []

alita_sdk/cli/testcases/discovery.py

@@ -0,0 +1,96 @@
+"""
+Test case discovery and filtering utilities.
+
+Handles finding test case files and filtering based on user selections.
+"""
+
+import logging
+from pathlib import Path
+from typing import List, Tuple
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def discover_test_case_files(
+    test_cases_dir: str,
+    test_case_files: Tuple[str, ...]
+) -> List[Path]:
+    """Discover and filter test case files based on user selection.
+    
+    Args:
+        test_cases_dir: Directory containing test case files
+        test_case_files: Specific test case files to execute (empty = all)
+        
+    Returns:
+        List of test case file paths
+    """
+    test_cases_path = Path(test_cases_dir)
+    
+    if test_case_files:
+        # User specified specific test case files
+        test_case_files_set = set(test_case_files)
+        all_test_cases = sorted(test_cases_path.rglob('TC-*.md'))
+        test_case_files_list = [
+            tc for tc in all_test_cases 
+            if tc.name in test_case_files_set
+        ]
+        
+        # Check if all specified files were found
+        found_names = {tc.name for tc in test_case_files_list}
+        not_found = test_case_files_set - found_names
+        if not_found:
+            console.print(f"[yellow]⚠ Warning: Test case files not found: {', '.join(not_found)}[/yellow]")
+    else:
+        # Execute all test cases
+        test_case_files_list = sorted(test_cases_path.rglob('TC-*.md'))
+    
+    return test_case_files_list
+
+
+def validate_test_case_files(
+    test_case_files_list: List[Path],
+    test_cases_dir: str,
+    test_case_files: Tuple[str, ...]
+) -> bool:
+    """Validate that test case files were found.
+    
+    Args:
+        test_case_files_list: List of discovered test case files
+        test_cases_dir: Directory that was searched
+        test_case_files: Specific files that were requested
+        
+    Returns:
+        True if files were found, False otherwise (prints warning)
+    """
+    if not test_case_files_list:
+        if test_case_files:
+            console.print(f"[yellow]No matching test case files found in {test_cases_dir}[/yellow]")
+        else:
+            console.print(f"[yellow]No test case files found in {test_cases_dir}[/yellow]")
+        return False
+    
+    return True
+
+
+def print_test_execution_header(
+    agent_name: str,
+    test_case_files_list: List[Path],
+    test_case_files: Tuple[str, ...],
+    results_dir: str
+) -> None:
+    """Print test execution header information.
+    
+    Args:
+        agent_name: Name of the test runner agent
+        test_case_files_list: List of test cases to execute
+        test_case_files: Specific files requested (if any)
+        results_dir: Directory where results will be saved
+    """
+    console.print(f"\n[bold cyan]🧪 Test Execution Started[/bold cyan]")
+    console.print(f"Agent: [bold]{agent_name}[/bold]")
+    console.print(f"Test Cases: {len(test_case_files_list)}")
+    if test_case_files:
+        console.print(f"Selected: [cyan]{', '.join(test_case_files)}[/cyan]")
+    console.print(f"Results Directory: {results_dir}\n")

alita_sdk/cli/testcases/executor.py

@@ -0,0 +1,84 @@
+"""
+Executor cache management for test execution.
+
+Handles creating, caching, and cleaning up agent executors.
+"""
+
+import logging
+import sqlite3
+from typing import Dict, Optional, Any, Tuple
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def cleanup_executor_cache(cache: Dict[str, tuple], cache_name: str = "executor") -> None:
+    """Clean up executor cache resources.
+    
+    Args:
+        cache: Dictionary of cached executors
+        cache_name: Name of cache for logging
+    """
+    console.print(f"[dim]Cleaning up {cache_name} cache...[/dim]")
+    for cache_key, cached_items in cache.items():
+        try:
+            # Extract memory from tuple (second element)
+            memory = cached_items[1] if len(cached_items) > 1 else None
+            
+            # Close SQLite memory connection
+            if memory and hasattr(memory, 'conn') and memory.conn:
+                memory.conn.close()
+        except Exception as e:
+            logger.debug(f"Error cleaning up {cache_name} cache for {cache_key}: {e}")
+
+
+def create_executor_from_cache(
+    cache: Dict[str, tuple], 
+    cache_key: str, 
+    client, 
+    agent_def: Dict[str, Any], 
+    toolkit_config_path: Optional[str],
+    config, 
+    model: Optional[str], 
+    temperature: Optional[float],
+    max_tokens: Optional[int], 
+    work_dir: Optional[str],
+    setup_executor_func
+) -> Tuple:
+    """Get or create executor from cache.
+    
+    Args:
+        cache: Executor cache dictionary
+        cache_key: Key for caching
+        client: API client
+        agent_def: Agent definition
+        toolkit_config_path: Path to toolkit config
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        setup_executor_func: Function to setup local agent executor
+        
+    Returns:
+        Tuple of (agent_executor, memory, mcp_session_manager)
+    """
+    if cache_key in cache:
+        return cache[cache_key]
+    
+    # Create new executor
+    from langgraph.checkpoint.sqlite import SqliteSaver
+    
+    memory = SqliteSaver(sqlite3.connect(":memory:", check_same_thread=False))
+    toolkit_config_tuple = (toolkit_config_path,) if toolkit_config_path else ()
+    
+    agent_executor, mcp_session_manager, _, _, _, _, _ = setup_executor_func(
+        client, agent_def, toolkit_config_tuple, config, model, temperature, 
+        max_tokens, memory, work_dir
+    )
+    
+    # Cache the executor
+    cached_tuple = (agent_executor, memory, mcp_session_manager)
+    cache[cache_key] = cached_tuple
+    return cached_tuple

alita_sdk/cli/testcases/logger.py

@@ -0,0 +1,85 @@
+"""
+Test execution logging utilities.
+
+Provides the TestLogCapture context manager for capturing console output to log files.
+"""
+
+from pathlib import Path
+from datetime import datetime
+from typing import Optional
+
+from rich.console import Console
+
+
+class TestLogCapture:
+    """
+    Context manager to capture console output to a log file.
+    Creates a single log file for the entire test execution session.
+    Strips Rich markup for plain text output with UTF-8 encoding.
+    """
+    
+    def __init__(self, results_dir: Path, session_name: str, console: Optional[Console] = None):
+        """
+        Initialize log capture.
+        
+        Args:
+            results_dir: Base results directory
+            session_name: Name for this test session
+        """
+        self.results_dir = results_dir
+        self.session_name = session_name
+        self.log_file_path = None
+        self.was_recording = False
+        # Use injected console if provided; otherwise create a dedicated console.
+        # Injecting allows other components (e.g., tool-call callbacks) to print to the
+        # same console and be captured in the session log.
+        self.console: Console = console or Console()
+        
+    def __enter__(self):
+        """Start capturing console output."""
+        # Extract toolkit name from session name (e.g., "test-execution-confluence" -> "confluence")
+        toolkit_name = self.session_name.replace('test-execution-', '')
+        
+        # Create toolkit-specific directory
+        toolkit_dir = self.results_dir / toolkit_name
+        toolkit_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Generate timestamped filename
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        log_filename = f"{timestamp}.{self.session_name}.txt"
+        self.log_file_path = toolkit_dir / log_filename
+        
+        # Enable recording on this console
+        self.was_recording = self.console.record
+        self.console.record = True
+        
+        return self
+    
+    def print(self, *args, **kwargs):
+        """Print to console (both display and recording)."""
+        self.console.print(*args, **kwargs)
+    
+    def status(self, message, **kwargs):
+        """Create status spinner and log the message."""
+        # Manually record the status message (spinners are transient)
+        self.console.print(message)
+        # Show spinner (this won't be recorded due to transient nature)
+        return self.console.status(message, **kwargs)
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Stop capturing and save log file."""
+        if self.log_file_path:
+            # Export recorded text (strips Rich markup)
+            plain_text = self.console.export_text()
+            
+            # Save to file with UTF-8 encoding
+            with open(self.log_file_path, 'w', encoding='utf-8') as f:
+                f.write(f"Test Execution Session: {self.session_name}\n")
+                f.write(f"Timestamp: {datetime.now().isoformat()}\n")
+                f.write("=" * 80 + "\n\n")
+                f.write(plain_text)
+            
+            # Restore previous recording state
+            self.console.record = self.was_recording
+        
+        return False

alita_sdk/cli/testcases/parser.py

@@ -0,0 +1,172 @@
+"""
+Test case parsing utilities.
+
+Handles parsing of test case markdown files and resolving configuration paths.
+"""
+
+import re
+from pathlib import Path
+from typing import Optional, Dict, Any
+
+
+def resolve_toolkit_config_path(config_path_str: str, test_file: Path, test_cases_dir: Path) -> Optional[str]:
+    """
+    Resolve toolkit configuration file path from test case.
+    
+    Tries multiple locations in order:
+    1. Absolute path
+    2. Relative to test case file directory
+    3. Relative to test cases directory
+    4. Relative to workspace root
+    
+    Args:
+        config_path_str: Config path from test case
+        test_file: Path to the test case file
+        test_cases_dir: Path to test cases directory
+        
+    Returns:
+        Absolute path to config file if found, None otherwise
+    """
+    if not config_path_str:
+        return None
+    
+    # Normalize path separators
+    config_path_str = config_path_str.replace('\\', '/')
+    
+    # Try absolute path first
+    config_path = Path(config_path_str)
+    if config_path.is_absolute() and config_path.exists():
+        return str(config_path)
+    
+    # Try relative to test case file directory
+    config_path = test_file.parent / config_path_str
+    if config_path.exists():
+        return str(config_path)
+    
+    # Try relative to test_cases_dir
+    config_path = test_cases_dir / config_path_str
+    if config_path.exists():
+        return str(config_path)
+    
+    # Try relative to workspace root
+    workspace_root = Path.cwd()
+    config_path = workspace_root / config_path_str
+    if config_path.exists():
+        return str(config_path)
+    
+    return None
+
+
+def parse_test_case(test_case_path: str) -> Dict[str, Any]:
+    """
+    Parse a test case markdown file to extract configuration, steps, and expectations.
+    
+    Args:
+        test_case_path: Path to the test case markdown file
+        
+    Returns:
+        Dictionary containing:
+        - name: Test case name
+        - objective: Test objective
+        - config_path: Path to toolkit config file
+        - generate_test_data: Boolean flag indicating if test data generation is needed (default: True)
+        - test_data_config: Dictionary of test data configuration from table
+        - prerequisites: Pre-requisites section text
+        - variables: List of variable placeholders found (e.g., {{TEST_PR_NUMBER}})
+        - steps: List of test steps with their descriptions
+        - expectations: List of expectations/assertions
+    """
+    path = Path(test_case_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Test case not found: {test_case_path}")
+    
+    content = path.read_text(encoding='utf-8')
+    
+    # Extract test case name from the first heading
+    name_match = re.search(r'^#\s+(.+)$', content, re.MULTILINE)
+    name = name_match.group(1) if name_match else path.stem
+    
+    # Extract objective
+    objective_match = re.search(r'##\s+Objective\s*\n\n(.+?)(?=\n\n##|\Z)', content, re.DOTALL)
+    objective = objective_match.group(1).strip() if objective_match else ""
+    
+    # Extract config path and generateTestData flag
+    config_section_match = re.search(r'##\s+Config\s*\n\n(.+?)(?=\n\n##|\Z)', content, re.DOTALL)
+    config_path = None
+    generate_test_data = True  # Default to True if not specified
+    
+    if config_section_match:
+        config_section = config_section_match.group(1)
+        # Extract path
+        path_match = re.search(r'path:\s*(.+?)(?=\n|$)', config_section, re.MULTILINE)
+        if path_match:
+            config_path = path_match.group(1).strip()
+        
+        # Extract generateTestData flag
+        gen_data_match = re.search(r'generateTestData\s*:\s*(true|false)', config_section, re.IGNORECASE)
+        if gen_data_match:
+            generate_test_data = gen_data_match.group(1).lower() == 'true'
+    
+    # Extract Test Data Configuration section as a raw fenced code block string
+    # NOTE: We intentionally store the entire section as a single string rather than parsing
+    # individual table rows. This preserves the original formatting for downstream tools
+    # which may prefer the raw markdown block.
+    test_data_config = None
+    config_section_match = re.search(r'##\s+Test Data Configuration\s*\n(.+?)(?=\n##|\Z)', content, re.DOTALL)
+    if config_section_match:
+        config_section = config_section_match.group(1).strip()
+        # Store as a fenced code block to make it clear this is a raw block of text
+        test_data_config = f"\n{config_section}\n"
+    
+    # Extract Pre-requisites section
+    prerequisites = ""
+    prereq_match = re.search(r'##\s+Pre-requisites\s*\n\n(.+?)(?=\n\n##|\Z)', content, re.DOTALL)
+    if prereq_match:
+        prerequisites = prereq_match.group(1).strip()
+    
+    # Find all variable placeholders ({{VARIABLE_NAME}})
+    variables = list(set(re.findall(r'\{\{([A-Z_]+)\}\}', content)))
+    
+    # Extract test steps and expectations
+    steps = []
+    expectations = []
+    
+    # Find all Step sections
+    step_pattern = r'###\s+Step\s+(\d+):\s+(.+?)\n\n(.+?)(?=\n\n###|\n\n##|\Z)'
+    for step_match in re.finditer(step_pattern, content, re.DOTALL):
+        step_num = step_match.group(1)
+        step_title = step_match.group(2).strip()
+        step_content = step_match.group(3).strip()
+        
+        # Extract the actual instruction (first paragraph before "Expectation:")
+        instruction_match = re.search(r'(.+?)(?=\n\n\*\*Expectation:\*\*|\Z)', step_content, re.DOTALL)
+        instruction = instruction_match.group(1).strip() if instruction_match else step_content
+        
+        # Extract expectation if present
+        expectation_match = re.search(r'\*\*Expectation:\*\*\s+(.+)', step_content, re.DOTALL)
+        expectation = expectation_match.group(1).strip() if expectation_match else None
+        
+        steps.append({
+            'number': int(step_num),
+            'title': step_title,
+            'instruction': instruction,
+            'expectation': expectation
+        })
+        
+        if expectation:
+            expectations.append({
+                'step': int(step_num),
+                'description': expectation
+            })
+    
+    return {
+        'name': name,
+        'objective': objective,
+        'config_path': config_path,
+        'generate_test_data': generate_test_data,
+        'test_data_config': test_data_config,
+        'prerequisites': prerequisites,
+        'variables': variables,
+        'steps': steps,
+        'expectations': expectations
+    }