alita-sdk 0.3.379__py3-none-any.whl → 0.3.627__py3-none-any.whl

alita_sdk/cli/testcases/executor.py

@@ -0,0 +1,84 @@
+"""
+Executor cache management for test execution.
+
+Handles creating, caching, and cleaning up agent executors.
+"""
+
+import logging
+import sqlite3
+from typing import Dict, Optional, Any, Tuple
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def cleanup_executor_cache(cache: Dict[str, tuple], cache_name: str = "executor") -> None:
+    """Clean up executor cache resources.
+    
+    Args:
+        cache: Dictionary of cached executors
+        cache_name: Name of cache for logging
+    """
+    console.print(f"[dim]Cleaning up {cache_name} cache...[/dim]")
+    for cache_key, cached_items in cache.items():
+        try:
+            # Extract memory from tuple (second element)
+            memory = cached_items[1] if len(cached_items) > 1 else None
+            
+            # Close SQLite memory connection
+            if memory and hasattr(memory, 'conn') and memory.conn:
+                memory.conn.close()
+        except Exception as e:
+            logger.debug(f"Error cleaning up {cache_name} cache for {cache_key}: {e}")
+
+
+def create_executor_from_cache(
+    cache: Dict[str, tuple], 
+    cache_key: str, 
+    client, 
+    agent_def: Dict[str, Any], 
+    toolkit_config_path: Optional[str],
+    config, 
+    model: Optional[str], 
+    temperature: Optional[float],
+    max_tokens: Optional[int], 
+    work_dir: Optional[str],
+    setup_executor_func
+) -> Tuple:
+    """Get or create executor from cache.
+    
+    Args:
+        cache: Executor cache dictionary
+        cache_key: Key for caching
+        client: API client
+        agent_def: Agent definition
+        toolkit_config_path: Path to toolkit config
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        setup_executor_func: Function to setup local agent executor
+        
+    Returns:
+        Tuple of (agent_executor, memory, mcp_session_manager)
+    """
+    if cache_key in cache:
+        return cache[cache_key]
+    
+    # Create new executor
+    from langgraph.checkpoint.sqlite import SqliteSaver
+    
+    memory = SqliteSaver(sqlite3.connect(":memory:", check_same_thread=False))
+    toolkit_config_tuple = (toolkit_config_path,) if toolkit_config_path else ()
+    
+    agent_executor, mcp_session_manager, _, _, _, _, _ = setup_executor_func(
+        client, agent_def, toolkit_config_tuple, config, model, temperature, 
+        max_tokens, memory, work_dir
+    )
+    
+    # Cache the executor
+    cached_tuple = (agent_executor, memory, mcp_session_manager)
+    cache[cache_key] = cached_tuple
+    return cached_tuple

alita_sdk/cli/testcases/logger.py

@@ -0,0 +1,85 @@
+"""
+Test execution logging utilities.
+
+Provides the TestLogCapture context manager for capturing console output to log files.
+"""
+
+from pathlib import Path
+from datetime import datetime
+from typing import Optional
+
+from rich.console import Console
+
+
+class TestLogCapture:
+    """
+    Context manager to capture console output to a log file.
+    Creates a single log file for the entire test execution session.
+    Strips Rich markup for plain text output with UTF-8 encoding.
+    """
+    
+    def __init__(self, results_dir: Path, session_name: str, console: Optional[Console] = None):
+        """
+        Initialize log capture.
+        
+        Args:
+            results_dir: Base results directory
+            session_name: Name for this test session
+        """
+        self.results_dir = results_dir
+        self.session_name = session_name
+        self.log_file_path = None
+        self.was_recording = False
+        # Use injected console if provided; otherwise create a dedicated console.
+        # Injecting allows other components (e.g., tool-call callbacks) to print to the
+        # same console and be captured in the session log.
+        self.console: Console = console or Console()
+        
+    def __enter__(self):
+        """Start capturing console output."""
+        # Extract toolkit name from session name (e.g., "test-execution-confluence" -> "confluence")
+        toolkit_name = self.session_name.replace('test-execution-', '')
+        
+        # Create toolkit-specific directory
+        toolkit_dir = self.results_dir / toolkit_name
+        toolkit_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Generate timestamped filename
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        log_filename = f"{timestamp}.{self.session_name}.txt"
+        self.log_file_path = toolkit_dir / log_filename
+        
+        # Enable recording on this console
+        self.was_recording = self.console.record
+        self.console.record = True
+        
+        return self
+    
+    def print(self, *args, **kwargs):
+        """Print to console (both display and recording)."""
+        self.console.print(*args, **kwargs)
+    
+    def status(self, message, **kwargs):
+        """Create status spinner and log the message."""
+        # Manually record the status message (spinners are transient)
+        self.console.print(message)
+        # Show spinner (this won't be recorded due to transient nature)
+        return self.console.status(message, **kwargs)
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Stop capturing and save log file."""
+        if self.log_file_path:
+            # Export recorded text (strips Rich markup)
+            plain_text = self.console.export_text()
+            
+            # Save to file with UTF-8 encoding
+            with open(self.log_file_path, 'w', encoding='utf-8') as f:
+                f.write(f"Test Execution Session: {self.session_name}\n")
+                f.write(f"Timestamp: {datetime.now().isoformat()}\n")
+                f.write("=" * 80 + "\n\n")
+                f.write(plain_text)
+            
+            # Restore previous recording state
+            self.console.record = self.was_recording
+        
+        return False

alita_sdk/cli/testcases/parser.py

@@ -0,0 +1,172 @@
+"""
+Test case parsing utilities.
+
+Handles parsing of test case markdown files and resolving configuration paths.
+"""
+
+import re
+from pathlib import Path
+from typing import Optional, Dict, Any
+
+
+def resolve_toolkit_config_path(config_path_str: str, test_file: Path, test_cases_dir: Path) -> Optional[str]:
+    """
+    Resolve toolkit configuration file path from test case.
+    
+    Tries multiple locations in order:
+    1. Absolute path
+    2. Relative to test case file directory
+    3. Relative to test cases directory
+    4. Relative to workspace root
+    
+    Args:
+        config_path_str: Config path from test case
+        test_file: Path to the test case file
+        test_cases_dir: Path to test cases directory
+        
+    Returns:
+        Absolute path to config file if found, None otherwise
+    """
+    if not config_path_str:
+        return None
+    
+    # Normalize path separators
+    config_path_str = config_path_str.replace('\\', '/')
+    
+    # Try absolute path first
+    config_path = Path(config_path_str)
+    if config_path.is_absolute() and config_path.exists():
+        return str(config_path)
+    
+    # Try relative to test case file directory
+    config_path = test_file.parent / config_path_str
+    if config_path.exists():
+        return str(config_path)
+    
+    # Try relative to test_cases_dir
+    config_path = test_cases_dir / config_path_str
+    if config_path.exists():
+        return str(config_path)
+    
+    # Try relative to workspace root
+    workspace_root = Path.cwd()
+    config_path = workspace_root / config_path_str
+    if config_path.exists():
+        return str(config_path)
+    
+    return None
+
+
+def parse_test_case(test_case_path: str) -> Dict[str, Any]:
+    """
+    Parse a test case markdown file to extract configuration, steps, and expectations.
+    
+    Args:
+        test_case_path: Path to the test case markdown file
+        
+    Returns:
+        Dictionary containing:
+        - name: Test case name
+        - objective: Test objective
+        - config_path: Path to toolkit config file
+        - generate_test_data: Boolean flag indicating if test data generation is needed (default: True)
+        - test_data_config: Dictionary of test data configuration from table
+        - prerequisites: Pre-requisites section text
+        - variables: List of variable placeholders found (e.g., {{TEST_PR_NUMBER}})
+        - steps: List of test steps with their descriptions
+        - expectations: List of expectations/assertions
+    """
+    path = Path(test_case_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Test case not found: {test_case_path}")
+    
+    content = path.read_text(encoding='utf-8')
+    
+    # Extract test case name from the first heading
+    name_match = re.search(r'^#\s+(.+)$', content, re.MULTILINE)
+    name = name_match.group(1) if name_match else path.stem
+    
+    # Extract objective
+    objective_match = re.search(r'##\s+Objective\s*\n\n(.+?)(?=\n\n##|\Z)', content, re.DOTALL)
+    objective = objective_match.group(1).strip() if objective_match else ""
+    
+    # Extract config path and generateTestData flag
+    config_section_match = re.search(r'##\s+Config\s*\n\n(.+?)(?=\n\n##|\Z)', content, re.DOTALL)
+    config_path = None
+    generate_test_data = True  # Default to True if not specified
+    
+    if config_section_match:
+        config_section = config_section_match.group(1)
+        # Extract path
+        path_match = re.search(r'path:\s*(.+?)(?=\n|$)', config_section, re.MULTILINE)
+        if path_match:
+            config_path = path_match.group(1).strip()
+        
+        # Extract generateTestData flag
+        gen_data_match = re.search(r'generateTestData\s*:\s*(true|false)', config_section, re.IGNORECASE)
+        if gen_data_match:
+            generate_test_data = gen_data_match.group(1).lower() == 'true'
+    
+    # Extract Test Data Configuration section as a raw fenced code block string
+    # NOTE: We intentionally store the entire section as a single string rather than parsing
+    # individual table rows. This preserves the original formatting for downstream tools
+    # which may prefer the raw markdown block.
+    test_data_config = None
+    config_section_match = re.search(r'##\s+Test Data Configuration\s*\n(.+?)(?=\n##|\Z)', content, re.DOTALL)
+    if config_section_match:
+        config_section = config_section_match.group(1).strip()
+        # Store as a fenced code block to make it clear this is a raw block of text
+        test_data_config = f"\n{config_section}\n"
+    
+    # Extract Pre-requisites section
+    prerequisites = ""
+    prereq_match = re.search(r'##\s+Pre-requisites\s*\n\n(.+?)(?=\n\n##|\Z)', content, re.DOTALL)
+    if prereq_match:
+        prerequisites = prereq_match.group(1).strip()
+    
+    # Find all variable placeholders ({{VARIABLE_NAME}})
+    variables = list(set(re.findall(r'\{\{([A-Z_]+)\}\}', content)))
+    
+    # Extract test steps and expectations
+    steps = []
+    expectations = []
+    
+    # Find all Step sections
+    step_pattern = r'###\s+Step\s+(\d+):\s+(.+?)\n\n(.+?)(?=\n\n###|\n\n##|\Z)'
+    for step_match in re.finditer(step_pattern, content, re.DOTALL):
+        step_num = step_match.group(1)
+        step_title = step_match.group(2).strip()
+        step_content = step_match.group(3).strip()
+        
+        # Extract the actual instruction (first paragraph before "Expectation:")
+        instruction_match = re.search(r'(.+?)(?=\n\n\*\*Expectation:\*\*|\Z)', step_content, re.DOTALL)
+        instruction = instruction_match.group(1).strip() if instruction_match else step_content
+        
+        # Extract expectation if present
+        expectation_match = re.search(r'\*\*Expectation:\*\*\s+(.+)', step_content, re.DOTALL)
+        expectation = expectation_match.group(1).strip() if expectation_match else None
+        
+        steps.append({
+            'number': int(step_num),
+            'title': step_title,
+            'instruction': instruction,
+            'expectation': expectation
+        })
+        
+        if expectation:
+            expectations.append({
+                'step': int(step_num),
+                'description': expectation
+            })
+    
+    return {
+        'name': name,
+        'objective': objective,
+        'config_path': config_path,
+        'generate_test_data': generate_test_data,
+        'test_data_config': test_data_config,
+        'prerequisites': prerequisites,
+        'variables': variables,
+        'steps': steps,
+        'expectations': expectations
+    }

alita_sdk/cli/testcases/prompts.py

@@ -0,0 +1,91 @@
+"""
+Prompt building utilities for test execution.
+
+Builds prompts for data generation, test execution, and validation.
+"""
+
+from typing import Dict, Any
+
+
+def build_bulk_data_gen_prompt(parsed_test_cases: list) -> str:
+    """Build consolidated requirements text for bulk test data generation."""
+    requirements = []
+    for idx, tc in enumerate(parsed_test_cases, 1):
+        test_case = tc['data']
+        test_file = tc['file']
+        # Build parts for this test case (do not include separator lines here;
+        # the entire block is wrapped with separators at the top-level)
+        parts = [f"Test Case #{idx}: {test_case['name']}", f"File: {test_file.name}", ""]
+        
+        if test_case.get('test_data_config'):
+            parts.append("Test Data Configuration:")
+            td = test_case['test_data_config']
+            raw_lines = str(td).splitlines()
+            for line in raw_lines:
+                parts.append(f"{line}")
+    
+        if test_case.get('prerequisites'):
+            parts.append(f"\nPre-requisites:\n{test_case['prerequisites']}")
+        
+        requirements.append("\n".join(parts))
+    
+    # If no requirements were collected, return an empty string to avoid
+    # producing a prompt with only separator lines.
+    if not requirements:
+        return ""
+
+    # Use a visible divider between test cases so each entry is clearly separated
+    divider = '-' * 40
+    body = f"\n\n{divider}\n\n".join(requirements)
+    return f"{('='*60)}\n\n{body}\n\n{('='*60)}"
+
+
+def build_single_test_execution_prompt(test_case_info: Dict[str, Any], test_number: int) -> str:
+    """Build execution prompt for a single test case."""
+    test_case = test_case_info['data']
+    test_file = test_case_info['file']
+    
+    parts = [
+        f"\n{'='*80}",
+        f"TEST CASE #{test_number}: {test_case['name']}",
+        f"File: {test_file.name}",
+        f"{'='*80}"
+    ]
+    
+    if test_case['steps']:
+        for step in test_case['steps']:
+            parts.append(f"\nStep {step['number']}: {step['title']}")
+            parts.append(step['instruction'])
+    else:
+        parts.append("\n(No steps defined)")
+    
+    return "\n".join(parts)
+
+
+def build_single_test_validation_prompt(test_case_info: Dict[str, Any], test_number: int, execution_output: str) -> str:
+    """Build validation prompt for a single test case."""
+    test_case = test_case_info['data']
+    
+    parts = [
+        f"\nTest Case #{test_number}: {test_case['name']}"
+    ]
+    
+    if test_case['steps']:
+        for step in test_case['steps']:
+            parts.append(f"  Step {step['number']}: {step['title']}")
+            if step['expectation']:
+                parts.append(f"    Expected: {step['expectation']}")
+    
+    parts.append(f"\n\nActual Execution Results:\n{execution_output}\n")
+    
+    # Escape quotes in test name for valid JSON in prompt
+    escaped_test_name = test_case['name'].replace('"', '\\"')
+    
+    parts.append(f"""\nBased on the execution results above, validate this test case.
+        {{
+          "test_number": {test_number},
+          "test_name": "{escaped_test_name}"
+        }}
+    """)
+    
+    return "\n".join(parts)

alita_sdk/cli/testcases/reporting.py

@@ -0,0 +1,125 @@
+"""
+Test result reporting and summary generation.
+
+Handles generating test reports and displaying summaries.
+"""
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Any
+from rich.console import Console
+from rich.table import Table
+from rich import box
+
+console = Console()
+
+
+def generate_summary_report(test_results: List[Dict[str, Any]]) -> Table:
+    """Generate a summary table for test results.
+    
+    Args:
+        test_results: List of test result dicts
+        
+    Returns:
+        Rich Table with summary statistics
+    """
+    total_tests = len(test_results)
+    passed_tests = sum(1 for r in test_results if r['passed'])
+    failed_tests = total_tests - passed_tests
+    pass_rate = (passed_tests / total_tests * 100) if total_tests > 0 else 0
+    
+    summary_table = Table(box=box.ROUNDED, border_style="cyan")
+    summary_table.add_column("Metric", style="bold")
+    summary_table.add_column("Value", justify="right")
+    
+    summary_table.add_row("Total Tests", str(total_tests))
+    summary_table.add_row("Passed", f"[green]{passed_tests}[/green]")
+    summary_table.add_row("Failed", f"[red]{failed_tests}[/red]")
+    summary_table.add_row("Pass Rate", f"{pass_rate:.1f}%")
+    
+    return summary_table
+
+
+def save_structured_report(
+    test_results: List[Dict[str, Any]],
+    results_dir: str,
+    log_file: Path = None
+) -> Path:
+    """Save structured JSON report of test results.
+    
+    Args:
+        test_results: List of test result dicts
+        results_dir: Directory to save report
+        log_file: Optional path to log file
+        
+    Returns:
+        Path to saved report file
+    """
+    results_path = Path(results_dir)
+    results_path.mkdir(parents=True, exist_ok=True)
+    
+    total_tests = len(test_results)
+    passed_tests = sum(1 for r in test_results if r['passed'])
+    failed_tests = total_tests - passed_tests
+    pass_rate = (passed_tests / total_tests * 100) if total_tests > 0 else 0
+    overall_result = "pass" if failed_tests == 0 else "fail"
+    
+    structured_report = {
+        "test_cases": [
+            {
+                "title": r['title'], 
+                "passed": r['passed'],
+                "steps": r.get('step_results', [])
+            } 
+            for r in test_results
+        ],
+        "overall_result": overall_result,
+        "summary": {
+            "total_tests": total_tests,
+            "passed": passed_tests,
+            "failed": failed_tests,
+            "pass_rate": f"{pass_rate:.1f}%"
+        },
+        "timestamp": datetime.now().isoformat(),
+        "log_file": str(log_file) if log_file else None
+    }
+    
+    summary_file = results_path / "test_execution_summary.json"
+    
+    console.print(f"\n[bold yellow]💾 Saving test execution summary...[/bold yellow]")
+    with open(summary_file, 'w') as f:
+        json.dump(structured_report, f, indent=2)
+    console.print(f"[green]✓ Summary saved to {summary_file}[/green]\n")
+    
+    return summary_file
+
+
+def print_test_execution_summary(
+    test_results: List[Dict[str, Any]],
+    results_dir: str,
+    session_name: str
+) -> None:
+    """Print test execution summary to console.
+    
+    Args:
+        test_results: List of test result dicts
+        results_dir: Directory where results are saved
+        session_name: Session name for finding log file
+    """
+    console.print(f"\n[bold]{'='*60}[/bold]")
+    console.print(f"[bold cyan]📊 Test Execution Summary[/bold cyan]")
+    console.print(f"[bold]{'='*60}[/bold]\n")
+    
+    summary_table = generate_summary_report(test_results)
+    console.print(summary_table)
+    
+    # Show log file location
+    results_path = Path(results_dir)
+    toolkit_name = session_name.replace('test-execution-', '')
+    toolkit_dir = results_path / toolkit_name
+    log_files = sorted(toolkit_dir.glob(f"*{session_name}.txt")) if toolkit_dir.exists() else []
+    
+    console.print(f"\n[bold cyan]📁 Log File[/bold cyan]")
+    if log_files:
+        console.print(f"  [dim]{log_files[0]}[/dim]")

alita_sdk/cli/testcases/setup.py

@@ -0,0 +1,108 @@
+"""
+Agent setup utilities for test execution.
+
+Handles loading and validating test runner, data generator, and validator agents.
+"""
+
+import logging
+from pathlib import Path
+from typing import Optional, Dict, Any, Tuple
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def load_test_runner_agent(agent_source: str) -> Tuple[Dict[str, Any], str]:
+    """Load test runner agent definition.
+    
+    Args:
+        agent_source: Path to agent definition file
+        
+    Returns:
+        Tuple of (agent_def, agent_name)
+        
+    Raises:
+        FileNotFoundError: If agent file doesn't exist
+    """
+    from ..agent_loader import load_agent_definition
+    
+    agent_source_path = Path(agent_source)
+    if not agent_source_path.exists():
+        raise FileNotFoundError(
+            f"Agent definition not found: {agent_source}. "
+            f"Make sure you are running from the repository root, "
+            f"or pass --agent_source explicitly."
+        )
+    
+    agent_def = load_agent_definition(agent_source)
+    agent_name = agent_def.get('name', agent_source_path.stem)
+    
+    return agent_def, agent_name
+
+
+def load_data_generator_agent(data_generator: str, skip_data_generation: bool) -> Optional[Dict[str, Any]]:
+    """Load data generator agent definition if needed.
+    
+    Args:
+        data_generator: Path to data generator agent file
+        skip_data_generation: Whether data generation is skipped
+        
+    Returns:
+        Agent definition dict or None if skipped/failed
+    """
+    from ..agent_loader import load_agent_definition
+    
+    if skip_data_generation:
+        return None
+    
+    if not data_generator:
+        return None
+    
+    try:
+        data_gen_def = load_agent_definition(data_generator)
+        data_gen_name = data_gen_def.get('name', Path(data_generator).stem)
+        console.print(f"Data Generator Agent: [bold]{data_gen_name}[/bold]\n")
+        return data_gen_def
+    except Exception as e:
+        console.print(f"[yellow]⚠ Warning: Failed to setup data generator: {e}[/yellow]")
+        console.print("[yellow]Continuing with test execution...[/yellow]\n")
+        logger.debug(f"Data generator setup error: {e}", exc_info=True)
+        return None
+
+
+def load_validator_agent(validator: Optional[str]) -> Tuple[Optional[Dict[str, Any]], str, Optional[str]]:
+    """Load validator agent definition.
+    
+    Args:
+        validator: Path to validator agent file (optional)
+        
+    Returns:
+        Tuple of (validator_def, validator_name, validator_path)
+    """
+    from ..agent_loader import load_agent_definition
+    
+    validator_def = None
+    validator_agent_name = "Default Validator"
+    validator_path = validator
+    
+    # Try to load validator from specified path or default location
+    if not validator_path:
+        default_validator = Path.cwd() / '.alita' / 'agents' / 'test-validator.agent.md'
+        if default_validator.exists():
+            validator_path = str(default_validator)
+    
+    if validator_path and Path(validator_path).exists():
+        try:
+            validator_def = load_agent_definition(validator_path)
+            validator_agent_name = validator_def.get('name', Path(validator_path).stem)
+            console.print(f"Validator Agent: [bold]{validator_agent_name}[/bold]")
+            console.print(f"[dim]Using: {validator_path}[/dim]\n")
+        except Exception as e:
+            console.print(f"[yellow]⚠ Warning: Failed to load validator agent: {e}[/yellow]")
+            console.print(f"[yellow]Will use test runner agent for validation[/yellow]\n")
+            logger.debug(f"Validator load error: {e}", exc_info=True)
+    else:
+        console.print(f"[dim]No validator agent specified, using test runner agent for validation[/dim]\n")
+    
+    return validator_def, validator_agent_name, validator_path