alita-sdk 0.3.462__py3-none-any.whl → 0.3.627__py3-none-any.whl

alita_sdk/cli/testcases/validation.py

@@ -0,0 +1,90 @@
+"""
+Validation utilities for test execution.
+
+Handles JSON extraction, fallback results, and diagnostics.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Any
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def extract_json_from_text(text: str) -> dict:
+    """Extract JSON object from text using brace counting."""
+    start_idx = text.find('{')
+    if start_idx == -1:
+        raise ValueError("No JSON found in text")
+    
+    brace_count = 0
+    end_idx = -1
+    for i, char in enumerate(text[start_idx:], start=start_idx):
+        if char == '{':
+            brace_count += 1
+        elif char == '}':
+            brace_count -= 1
+            if brace_count == 0:
+                end_idx = i + 1
+                break
+    
+    if end_idx == -1:
+        raise ValueError("Could not find matching closing brace")
+    
+    return json.loads(text[start_idx:end_idx])
+
+
+def create_fallback_result_for_test(test_case: Dict[str, Any], test_file: Path, reason: str = 'Validation failed') -> Dict[str, Any]:
+    """Create a fallback result for a single test case with detailed step information.
+    
+    Args:
+        test_case: Parsed test case data
+        test_file: Path to test case file
+        reason: Reason for fallback
+        
+    Returns:
+        Fallback test result dict with step details
+    """
+    fallback_steps = []
+    for step_info in test_case.get('steps', []):
+        fallback_steps.append({
+            'step_number': step_info['number'],
+            'title': step_info['title'],
+            'passed': False,
+            'details': reason
+        })
+    
+    return {
+        'title': test_case['name'],
+        'passed': False,
+        'file': test_file.name,
+        'step_results': fallback_steps,
+        'validation_error': reason
+    }
+
+
+def print_validation_diagnostics(validation_output: str) -> None:
+    """Print diagnostic information for validation output.
+    
+    Args:
+        validation_output: The validation output to diagnose
+    """
+    console.print(f"\n[bold red]🔍 Diagnostic Information:[/bold red]")
+    console.print(f"[dim]Output length: {len(validation_output)} characters[/dim]")
+    
+    # Check for key JSON elements
+    has_json = '{' in validation_output and '}' in validation_output
+    has_fields = 'test_number' in validation_output and 'steps' in validation_output
+    
+    console.print(f"[dim]Has JSON structure: {has_json}[/dim]")
+    console.print(f"[dim]Has required fields: {has_fields}[/dim]")
+    
+    # Show relevant excerpt
+    if len(validation_output) > 400:
+        console.print(f"\n[red]First 200 chars:[/red] [dim]{validation_output[:200]}[/dim]")
+        console.print(f"[red]Last 200 chars:[/red] [dim]{validation_output[-200:]}[/dim]")
+    else:
+        console.print(f"\n[red]Full output:[/red] [dim]{validation_output}[/dim]")

alita_sdk/cli/testcases/workflow.py

@@ -0,0 +1,196 @@
+"""
+Main workflow orchestration for test case execution.
+
+Coordinates the entire test execution flow from parsing to reporting.
+"""
+
+import logging
+import uuid
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+def parse_all_test_cases(
+    test_case_files_list: List[Path],
+    master_log
+) -> List[Dict[str, Any]]:
+    """Parse all test case files.
+    
+    Args:
+        test_case_files_list: List of test case files to parse
+        master_log: Log capture instance
+        
+    Returns:
+        List of parsed test case dicts with 'file' and 'data' keys
+    """
+    from .parser import parse_test_case
+    
+    parsed_test_cases = []
+    for test_file in test_case_files_list:
+        try:
+            test_case = parse_test_case(str(test_file))
+            parsed_test_cases.append({
+                'file': test_file,
+                'data': test_case
+            })
+        except Exception as e:
+            master_log.print(f"[yellow]⚠ Warning: Failed to parse {test_file.name}: {e}[/yellow]")
+            logger.debug(f"Parse error for {test_file.name}: {e}", exc_info=True)
+    
+    return parsed_test_cases
+
+
+def filter_test_cases_needing_data_gen(
+    parsed_test_cases: List[Dict[str, Any]]
+) -> List[Dict[str, Any]]:
+    """Filter test cases that need data generation.
+    
+    Args:
+        parsed_test_cases: All parsed test cases
+        
+    Returns:
+        Filtered list of test cases that require data generation
+    """
+    return [
+        tc for tc in parsed_test_cases 
+        if tc['data'].get('generate_test_data', True)
+    ]
+
+
+def execute_all_test_cases(
+    parsed_test_cases: List[Dict[str, Any]],
+    bulk_gen_chat_history: List[Dict[str, str]],
+    test_cases_path: Path,
+    agent_def: Dict[str, Any],
+    validator_def: Optional[Dict[str, Any]],
+    client,
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> List[Dict[str, Any]]:
+    """Execute all test cases and return results.
+    
+    Args:
+        parsed_test_cases: List of parsed test cases
+        bulk_gen_chat_history: Chat history from data generation
+        test_cases_path: Path to test cases directory
+        agent_def: Test runner agent definition
+        validator_def: Validator agent definition (optional)
+        client: API client
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        List of test result dicts
+    """
+    from .parser import resolve_toolkit_config_path
+    from .utils import extract_toolkit_name
+    from .executor import cleanup_executor_cache
+    from .test_runner import execute_single_test_case, validate_single_test_case
+    from .validation import create_fallback_result_for_test
+    
+    if not parsed_test_cases:
+        master_log.print("[yellow]No test cases to execute[/yellow]")
+        return []
+    
+    master_log.print(f"\n[bold yellow]📋 Executing test cases sequentially...[/bold yellow]\n")
+    
+    # Show data generation context availability
+    if bulk_gen_chat_history:
+        master_log.print(f"[dim]✓ Data generation history available ({len(bulk_gen_chat_history)} messages) - shared with all test cases[/dim]\n")
+    else:
+        master_log.print(f"[dim]ℹ No data generation history (skipped or disabled)[/dim]\n")
+    
+    # Executor caches
+    executor_cache = {}
+    validation_executor_cache = {}
+    
+    # Execute each test case sequentially
+    test_results = []
+    total_tests = len(parsed_test_cases)
+    
+    for idx, tc_info in enumerate(parsed_test_cases, 1):
+        test_case = tc_info['data']
+        test_file = tc_info['file']
+        test_name = test_case['name']
+        
+        try:
+            # Resolve toolkit config path
+            toolkit_config_path = resolve_toolkit_config_path(
+                test_case.get('config_path', ''),
+                test_file,
+                test_cases_path
+            )
+            
+            # Use cache key
+            cache_key = toolkit_config_path if toolkit_config_path else '__no_config__'
+            
+            # Execute single test case
+            execution_output = execute_single_test_case(
+                tc_info, idx, total_tests, bulk_gen_chat_history, test_cases_path,
+                executor_cache, client, agent_def, config, model, temperature,
+                max_tokens, work_dir, master_log, setup_executor_func,
+                verbose=verbose,
+                debug=debug,
+            )
+            
+            if not execution_output:
+                # Create fallback result for failed execution
+                test_results.append({
+                    'title': test_name,
+                    'passed': False,
+                    'file': test_file.name,
+                    'step_results': []
+                })
+                continue
+            
+            # Append execution to history for validation
+            from .prompts import build_single_test_execution_prompt
+            validation_chat_history = bulk_gen_chat_history + [
+                {"role": "user", "content": build_single_test_execution_prompt(tc_info, idx)},
+                {"role": "assistant", "content": execution_output}
+            ]
+            
+            # Validate test case
+            test_result = validate_single_test_case(
+                tc_info, idx, execution_output, validation_chat_history,
+                validation_executor_cache, cache_key, client, validator_def,
+                agent_def, toolkit_config_path, config, model, temperature,
+                max_tokens, work_dir, master_log, setup_executor_func,
+                verbose=verbose,
+                debug=debug,
+            )
+            
+            test_results.append(test_result)
+            
+        except Exception as e:
+            logger.debug(f"Test execution failed for {test_name}: {e}", exc_info=True)
+            master_log.print(f"[red]✗ Test execution failed: {e}[/red]")
+            
+            # Create fallback result
+            fallback_result = create_fallback_result_for_test(
+                test_case,
+                test_file,
+                f'Test execution failed: {str(e)}'
+            )
+            test_results.append(fallback_result)
+            master_log.print()
+    
+    # Cleanup executor caches
+    cleanup_executor_cache(executor_cache, "executor")
+    cleanup_executor_cache(validation_executor_cache, "validation executor")
+    
+    return test_results

alita_sdk/cli/toolkit.py

@@ -10,6 +10,7 @@ import logging
 from typing import Optional, Dict, Any
 
 from .cli import get_client
+from .toolkit_loader import load_toolkit_config
 
 logger = logging.getLogger(__name__)
 
@@ -140,24 +141,15 @@ def toolkit_schema(ctx, toolkit_name: str):
 @click.pass_context
 def toolkit_test(ctx, toolkit_type: str, tool: str, config_file, params, 
                 param, llm_model: str, temperature: float, max_tokens: int):
-    """
-    Test a specific tool from a toolkit.
+    """Test a specific tool from a toolkit.
     
     TOOLKIT_TYPE: Type of toolkit (e.g., 'jira', 'github', 'confluence')
     
+    \b
     Examples:
-    
-        # Test with config and params from files
-        alita-cli toolkit test jira --tool get_issue \\
-            --config jira-config.json --params params.json
-        
-        # Test with inline parameters
-        alita-cli toolkit test jira --tool get_issue \\
-            --config jira-config.json --param issue_key=PROJ-123
-        
-        # Test with JSON output for scripting
-        alita-cli --output json toolkit test github --tool get_issue \\
-            --config github-config.json --param owner=user --param repo=myrepo
+      alita toolkit test jira --tool get_issue --config jira.json --params params.json
+      alita toolkit test jira --tool get_issue --config jira.json --param issue_key=PROJ-123
+      alita -o json toolkit test github --tool get_issue --config github.json
     """
     formatter = ctx.obj['formatter']
     client = get_client(ctx)
@@ -166,9 +158,15 @@ def toolkit_test(ctx, toolkit_type: str, tool: str, config_file, params,
         # Load toolkit configuration
         toolkit_config = {}
         if config_file:
-            toolkit_config = json.load(config_file)
+            toolkit_config = load_toolkit_config(config_file.name)
             logger.debug(f"Loaded toolkit config from {config_file.name}")
         
+        # Add the tool to selected_tools in the config
+        if 'selected_tools' not in toolkit_config:
+            toolkit_config['selected_tools'] = []
+        if tool not in toolkit_config['selected_tools']:
+            toolkit_config['selected_tools'].append(tool)
+        
         # Load tool parameters
         tool_params = {}
         if params:
@@ -204,7 +202,6 @@ def toolkit_test(ctx, toolkit_type: str, tool: str, config_file, params,
         llm_config = {
             'temperature': temperature,
             'max_tokens': max_tokens,
-            'top_p': 1.0
         }
         
         # Execute test
@@ -254,7 +251,7 @@ def toolkit_tools(ctx, toolkit_type: str, config_file):
         # Load toolkit configuration if provided
         toolkit_config = {}
         if config_file:
-            toolkit_config = json.load(config_file)
+            toolkit_config = load_toolkit_config(config_file.name)
         
         # Import and instantiate toolkit
         from alita_sdk.tools import AVAILABLE_TOOLS

alita_sdk/cli/toolkit_loader.py

@@ -12,6 +12,20 @@ from typing import Dict, Any, List
 from .config import substitute_env_vars
 
 
+# All available tools in the inventory toolkit
+INVENTORY_TOOLS = [
+    "search_graph",
+    "get_entity",
+    "get_entity_content",
+    "impact_analysis",
+    "get_related_entities",
+    "get_stats",
+    "get_citations",
+    "list_entities_by_type",
+    "list_entities_by_layer",
+]
+
+
 def load_toolkit_config(file_path: str) -> Dict[str, Any]:
     """Load toolkit configuration from JSON or YAML file with env var substitution."""
     path = Path(file_path)
@@ -33,7 +47,15 @@ def load_toolkit_config(file_path: str) -> Dict[str, Any]:
 
 
 def load_toolkit_configs(agent_def: Dict[str, Any], toolkit_config_paths: tuple) -> List[Dict[str, Any]]:
-    """Load all toolkit configurations from agent definition and CLI options."""
+    """Load all toolkit configurations from agent definition and CLI options.
+    
+    Args:
+        agent_def: Agent definition dictionary
+        toolkit_config_paths: Tuple of file paths or dict configs from CLI options
+        
+    Returns:
+        List of toolkit configuration dictionaries
+    """
     toolkit_configs = []
     
     # Load from agent definition if present
@@ -45,11 +67,19 @@ def load_toolkit_configs(agent_def: Dict[str, Any], toolkit_config_paths: tuple)
                     toolkit_configs.append(config)
                 elif 'config' in tk_config:
                     toolkit_configs.append(tk_config['config'])
+                elif 'type' in tk_config:
+                    # Direct toolkit config (e.g., {'type': 'inventory', ...})
+                    toolkit_configs.append(tk_config)
     
-    # Load from CLI options
+    # Load from CLI options - can be file paths (str) or dict configs
     if toolkit_config_paths:
-        for config_path in toolkit_config_paths:
-            config = load_toolkit_config(config_path)
-            toolkit_configs.append(config)
+        for config_item in toolkit_config_paths:
+            if isinstance(config_item, dict):
+                # Direct config dict (e.g., from /inventory command)
+                toolkit_configs.append(config_item)
+            elif isinstance(config_item, str):
+                # File path
+                config = load_toolkit_config(config_item)
+                toolkit_configs.append(config)
     
     return toolkit_configs

alita_sdk/cli/tools/__init__.py

@@ -4,6 +4,40 @@ CLI tools package.
 Contains specialized tools for CLI agents.
 """
 
-from .filesystem import get_filesystem_tools
+from .filesystem import get_filesystem_tools, FilesystemApiWrapper
+from .terminal import get_terminal_tools, create_default_blocked_patterns_file
+from .planning import (
+    get_planning_tools, 
+    PlanState, 
+    list_sessions, 
+    generate_session_id,
+    create_session_memory,
+    save_session_metadata,
+    load_session_metadata,
+    update_session_metadata,
+    get_session_dir,
+    to_portable_path,
+    from_portable_path,
+)
+from .approval import create_approval_wrapper, ApprovalToolWrapper, prompt_approval
 
-__all__ = ['get_filesystem_tools']
+__all__ = [
+    'get_filesystem_tools',
+    'FilesystemApiWrapper',
+    'get_terminal_tools',
+    'create_default_blocked_patterns_file',
+    'get_planning_tools',
+    'PlanState',
+    'list_sessions',
+    'generate_session_id',
+    'create_session_memory',
+    'save_session_metadata',
+    'load_session_metadata',
+    'update_session_metadata',
+    'get_session_dir',
+    'to_portable_path',
+    'from_portable_path',
+    'create_approval_wrapper',
+    'ApprovalToolWrapper',
+    'prompt_approval',
+]

alita_sdk/cli/tools/approval.py

@@ -0,0 +1,224 @@
+"""
+Tool approval wrapper for CLI.
+
+Wraps tools to require user approval before execution based on approval mode.
+Modes:
+- 'always': Always require approval for each tool call
+- 'auto': No approval required (automatic execution)
+- 'yolo': No approval and no safety checks (use with caution)
+"""
+
+import functools
+from typing import Any, Callable, Dict, List, Optional, Set
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich import box
+from rich.text import Text
+import json
+
+from langchain_core.tools import BaseTool, StructuredTool, Tool
+
+console = Console()
+
+# Tools that always require approval in 'always' mode (dangerous built-in operations)
+# These are the built-in CLI tools that modify the filesystem or execute commands
+DANGEROUS_TOOLS = {
+    'terminal_run_command',  # Shell command execution
+    'write_file',
+    'create_file', 
+    'delete_file',
+    'move_file',
+    'copy_file',
+    'create_directory',
+}
+
+# Note: Tools NOT in DANGEROUS_TOOLS are auto-approved, including:
+# - Read-only filesystem tools (read_file, list_directory, etc.)
+# - User-added toolkit tools (via --toolkit_config or /add_toolkit)
+# - MCP server tools (via --mcp or /add_mcp)
+# 
+# The assumption is that users explicitly add toolkits they trust.
+# Only built-in tools that can modify files or run commands require approval.
+
+
+def prompt_approval(tool_name: str, tool_args: Dict[str, Any], approval_mode: str) -> bool:
+    """
+    Prompt user for approval before executing a tool.
+    
+    Args:
+        tool_name: Name of the tool to execute
+        tool_args: Arguments to pass to the tool
+        approval_mode: Current approval mode ('always', 'auto', 'yolo')
+        
+    Returns:
+        True if approved, False if rejected
+    """
+    # Auto mode - always approve
+    if approval_mode == 'auto':
+        return True
+    
+    # Yolo mode - always approve, no questions asked
+    if approval_mode == 'yolo':
+        return True
+    
+    # Always mode - only prompt for dangerous built-in tools
+    # User-added toolkits and MCP tools are auto-approved (user explicitly added them)
+    if tool_name not in DANGEROUS_TOOLS:
+        return True
+    
+    # Create approval prompt panel for dangerous tools
+    console.print()
+    
+    # Build args display
+    args_content = []
+    for key, value in tool_args.items():
+        if isinstance(value, str) and len(value) > 100:
+            display_value = value[:100] + "..."
+        elif isinstance(value, (dict, list)):
+            try:
+                formatted = json.dumps(value, indent=2)
+                if len(formatted) > 200:
+                    formatted = formatted[:200] + "..."
+                display_value = formatted
+            except:
+                display_value = str(value)[:100]
+        else:
+            display_value = str(value)
+        args_content.append(f"  [cyan]{key}[/cyan]: {display_value}")
+    
+    args_text = "\n".join(args_content) if args_content else "  (no arguments)"
+    
+    # All tools reaching here are dangerous (in DANGEROUS_TOOLS)
+    icon = "⚠️"
+    border_style = "yellow"
+    title_style = "bold yellow"
+    
+    panel = Panel(
+        Text.from_markup(f"[bold]{tool_name}[/bold]\n\n[dim]Arguments:[/dim]\n{args_text}"),
+        title=f"[{title_style}]{icon} Approve Tool Call?[/{title_style}]",
+        title_align="left",
+        subtitle="[dim][y]es / [n]o / [a]uto-approve / [q]uit[/dim]",
+        subtitle_align="right",
+        border_style=border_style,
+        box=box.ROUNDED,
+        padding=(1, 2),
+    )
+    console.print(panel)
+    
+    # Get user input
+    while True:
+        try:
+            response = input("→ ").strip().lower()
+        except (KeyboardInterrupt, EOFError):
+            console.print("\n[yellow]Cancelled[/yellow]")
+            return False
+        
+        if response in ('y', 'yes', ''):
+            console.print("[green]✓ Approved[/green]")
+            return True
+        elif response in ('n', 'no'):
+            console.print("[red]✗ Rejected[/red]")
+            return False
+        elif response in ('a', 'auto'):
+            console.print("[cyan]→ Switching to auto mode for this session[/cyan]")
+            # Signal to switch to auto mode
+            return 'switch_auto'
+        elif response in ('q', 'quit'):
+            console.print("[yellow]Quitting...[/yellow]")
+            raise KeyboardInterrupt()
+        else:
+            console.print("[dim]Please enter y/n/a/q[/dim]")
+
+
+class ApprovalToolWrapper:
+    """
+    Wrapper that adds approval prompts to tools based on approval mode.
+    
+    This wrapper intercepts tool calls and prompts for user approval
+    before executing dangerous operations.
+    """
+    
+    def __init__(self, approval_mode_ref: List[str]):
+        """
+        Initialize the approval wrapper.
+        
+        Args:
+            approval_mode_ref: A mutable list containing the current approval mode.
+                              Using a list allows the mode to be changed externally.
+                              access as approval_mode_ref[0]
+        """
+        self.approval_mode_ref = approval_mode_ref
+    
+    @property
+    def approval_mode(self) -> str:
+        """Get current approval mode."""
+        return self.approval_mode_ref[0] if self.approval_mode_ref else 'always'
+    
+    def wrap_tool(self, tool: BaseTool) -> BaseTool:
+        """
+        Wrap a tool to add approval prompts.
+        
+        Args:
+            tool: The tool to wrap
+            
+        Returns:
+            Wrapped tool with approval logic
+        """
+        original_func = tool.func if hasattr(tool, 'func') else tool._run
+        tool_name = tool.name
+        wrapper_self = self
+        
+        @functools.wraps(original_func)
+        def wrapped_func(*args, **kwargs):
+            # Get approval
+            approval = prompt_approval(tool_name, kwargs, wrapper_self.approval_mode)
+            
+            if approval == 'switch_auto':
+                # Switch to auto mode
+                wrapper_self.approval_mode_ref[0] = 'auto'
+                console.print("[cyan]Mode switched to 'auto' - tools will auto-approve[/cyan]")
+            elif not approval:
+                return f"Tool execution rejected by user"
+            
+            # Execute the tool
+            return original_func(*args, **kwargs)
+        
+        # Create new tool with wrapped function
+        if isinstance(tool, StructuredTool):
+            return StructuredTool(
+                name=tool.name,
+                description=tool.description,
+                func=wrapped_func,
+                args_schema=tool.args_schema,
+                return_direct=tool.return_direct,
+            )
+        else:
+            # Clone the tool with wrapped function
+            tool.func = wrapped_func
+            return tool
+    
+    def wrap_tools(self, tools: List[BaseTool]) -> List[BaseTool]:
+        """
+        Wrap multiple tools with approval logic.
+        
+        Args:
+            tools: List of tools to wrap
+            
+        Returns:
+            List of wrapped tools
+        """
+        return [self.wrap_tool(tool) for tool in tools]
+
+
+def create_approval_wrapper(approval_mode_ref: List[str]) -> ApprovalToolWrapper:
+    """
+    Create an approval wrapper with a reference to the current mode.
+    
+    Args:
+        approval_mode_ref: Mutable list containing current approval mode [0]
+        
+    Returns:
+        ApprovalToolWrapper instance
+    """
+    return ApprovalToolWrapper(approval_mode_ref)