alita-sdk 0.3.602__py3-none-any.whl → 0.3.609__py3-none-any.whl

alita_sdk/cli/testcases/validation.py

@@ -0,0 +1,90 @@
+"""
+Validation utilities for test execution.
+
+Handles JSON extraction, fallback results, and diagnostics.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Any
+from rich.console import Console
+
+logger = logging.getLogger(__name__)
+console = Console()
+
+
+def extract_json_from_text(text: str) -> dict:
+    """Extract JSON object from text using brace counting."""
+    start_idx = text.find('{')
+    if start_idx == -1:
+        raise ValueError("No JSON found in text")
+    
+    brace_count = 0
+    end_idx = -1
+    for i, char in enumerate(text[start_idx:], start=start_idx):
+        if char == '{':
+            brace_count += 1
+        elif char == '}':
+            brace_count -= 1
+            if brace_count == 0:
+                end_idx = i + 1
+                break
+    
+    if end_idx == -1:
+        raise ValueError("Could not find matching closing brace")
+    
+    return json.loads(text[start_idx:end_idx])
+
+
+def create_fallback_result_for_test(test_case: Dict[str, Any], test_file: Path, reason: str = 'Validation failed') -> Dict[str, Any]:
+    """Create a fallback result for a single test case with detailed step information.
+    
+    Args:
+        test_case: Parsed test case data
+        test_file: Path to test case file
+        reason: Reason for fallback
+        
+    Returns:
+        Fallback test result dict with step details
+    """
+    fallback_steps = []
+    for step_info in test_case.get('steps', []):
+        fallback_steps.append({
+            'step_number': step_info['number'],
+            'title': step_info['title'],
+            'passed': False,
+            'details': reason
+        })
+    
+    return {
+        'title': test_case['name'],
+        'passed': False,
+        'file': test_file.name,
+        'step_results': fallback_steps,
+        'validation_error': reason
+    }
+
+
+def print_validation_diagnostics(validation_output: str) -> None:
+    """Print diagnostic information for validation output.
+    
+    Args:
+        validation_output: The validation output to diagnose
+    """
+    console.print(f"\n[bold red]🔍 Diagnostic Information:[/bold red]")
+    console.print(f"[dim]Output length: {len(validation_output)} characters[/dim]")
+    
+    # Check for key JSON elements
+    has_json = '{' in validation_output and '}' in validation_output
+    has_fields = 'test_number' in validation_output and 'steps' in validation_output
+    
+    console.print(f"[dim]Has JSON structure: {has_json}[/dim]")
+    console.print(f"[dim]Has required fields: {has_fields}[/dim]")
+    
+    # Show relevant excerpt
+    if len(validation_output) > 400:
+        console.print(f"\n[red]First 200 chars:[/red] [dim]{validation_output[:200]}[/dim]")
+        console.print(f"[red]Last 200 chars:[/red] [dim]{validation_output[-200:]}[/dim]")
+    else:
+        console.print(f"\n[red]Full output:[/red] [dim]{validation_output}[/dim]")

alita_sdk/cli/testcases/workflow.py

@@ -0,0 +1,196 @@
+"""
+Main workflow orchestration for test case execution.
+
+Coordinates the entire test execution flow from parsing to reporting.
+"""
+
+import logging
+import uuid
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+def parse_all_test_cases(
+    test_case_files_list: List[Path],
+    master_log
+) -> List[Dict[str, Any]]:
+    """Parse all test case files.
+    
+    Args:
+        test_case_files_list: List of test case files to parse
+        master_log: Log capture instance
+        
+    Returns:
+        List of parsed test case dicts with 'file' and 'data' keys
+    """
+    from .parser import parse_test_case
+    
+    parsed_test_cases = []
+    for test_file in test_case_files_list:
+        try:
+            test_case = parse_test_case(str(test_file))
+            parsed_test_cases.append({
+                'file': test_file,
+                'data': test_case
+            })
+        except Exception as e:
+            master_log.print(f"[yellow]⚠ Warning: Failed to parse {test_file.name}: {e}[/yellow]")
+            logger.debug(f"Parse error for {test_file.name}: {e}", exc_info=True)
+    
+    return parsed_test_cases
+
+
+def filter_test_cases_needing_data_gen(
+    parsed_test_cases: List[Dict[str, Any]]
+) -> List[Dict[str, Any]]:
+    """Filter test cases that need data generation.
+    
+    Args:
+        parsed_test_cases: All parsed test cases
+        
+    Returns:
+        Filtered list of test cases that require data generation
+    """
+    return [
+        tc for tc in parsed_test_cases 
+        if tc['data'].get('generate_test_data', True)
+    ]
+
+
+def execute_all_test_cases(
+    parsed_test_cases: List[Dict[str, Any]],
+    bulk_gen_chat_history: List[Dict[str, str]],
+    test_cases_path: Path,
+    agent_def: Dict[str, Any],
+    validator_def: Optional[Dict[str, Any]],
+    client,
+    config,
+    model: Optional[str],
+    temperature: Optional[float],
+    max_tokens: Optional[int],
+    work_dir: str,
+    master_log,
+    setup_executor_func,
+    verbose: bool = True,
+    debug: bool = False,
+) -> List[Dict[str, Any]]:
+    """Execute all test cases and return results.
+    
+    Args:
+        parsed_test_cases: List of parsed test cases
+        bulk_gen_chat_history: Chat history from data generation
+        test_cases_path: Path to test cases directory
+        agent_def: Test runner agent definition
+        validator_def: Validator agent definition (optional)
+        client: API client
+        config: CLI configuration
+        model: Model override
+        temperature: Temperature override
+        max_tokens: Max tokens override
+        work_dir: Working directory
+        master_log: Log capture instance
+        setup_executor_func: Function to setup executor
+        
+    Returns:
+        List of test result dicts
+    """
+    from .parser import resolve_toolkit_config_path
+    from .utils import extract_toolkit_name
+    from .executor import cleanup_executor_cache
+    from .test_runner import execute_single_test_case, validate_single_test_case
+    from .validation import create_fallback_result_for_test
+    
+    if not parsed_test_cases:
+        master_log.print("[yellow]No test cases to execute[/yellow]")
+        return []
+    
+    master_log.print(f"\n[bold yellow]📋 Executing test cases sequentially...[/bold yellow]\n")
+    
+    # Show data generation context availability
+    if bulk_gen_chat_history:
+        master_log.print(f"[dim]✓ Data generation history available ({len(bulk_gen_chat_history)} messages) - shared with all test cases[/dim]\n")
+    else:
+        master_log.print(f"[dim]ℹ No data generation history (skipped or disabled)[/dim]\n")
+    
+    # Executor caches
+    executor_cache = {}
+    validation_executor_cache = {}
+    
+    # Execute each test case sequentially
+    test_results = []
+    total_tests = len(parsed_test_cases)
+    
+    for idx, tc_info in enumerate(parsed_test_cases, 1):
+        test_case = tc_info['data']
+        test_file = tc_info['file']
+        test_name = test_case['name']
+        
+        try:
+            # Resolve toolkit config path
+            toolkit_config_path = resolve_toolkit_config_path(
+                test_case.get('config_path', ''),
+                test_file,
+                test_cases_path
+            )
+            
+            # Use cache key
+            cache_key = toolkit_config_path if toolkit_config_path else '__no_config__'
+            
+            # Execute single test case
+            execution_output = execute_single_test_case(
+                tc_info, idx, total_tests, bulk_gen_chat_history, test_cases_path,
+                executor_cache, client, agent_def, config, model, temperature,
+                max_tokens, work_dir, master_log, setup_executor_func,
+                verbose=verbose,
+                debug=debug,
+            )
+            
+            if not execution_output:
+                # Create fallback result for failed execution
+                test_results.append({
+                    'title': test_name,
+                    'passed': False,
+                    'file': test_file.name,
+                    'step_results': []
+                })
+                continue
+            
+            # Append execution to history for validation
+            from .prompts import build_single_test_execution_prompt
+            validation_chat_history = bulk_gen_chat_history + [
+                {"role": "user", "content": build_single_test_execution_prompt(tc_info, idx)},
+                {"role": "assistant", "content": execution_output}
+            ]
+            
+            # Validate test case
+            test_result = validate_single_test_case(
+                tc_info, idx, execution_output, validation_chat_history,
+                validation_executor_cache, cache_key, client, validator_def,
+                agent_def, toolkit_config_path, config, model, temperature,
+                max_tokens, work_dir, master_log, setup_executor_func,
+                verbose=verbose,
+                debug=debug,
+            )
+            
+            test_results.append(test_result)
+            
+        except Exception as e:
+            logger.debug(f"Test execution failed for {test_name}: {e}", exc_info=True)
+            master_log.print(f"[red]✗ Test execution failed: {e}[/red]")
+            
+            # Create fallback result
+            fallback_result = create_fallback_result_for_test(
+                test_case,
+                test_file,
+                f'Test execution failed: {str(e)}'
+            )
+            test_results.append(fallback_result)
+            master_log.print()
+    
+    # Cleanup executor caches
+    cleanup_executor_cache(executor_cache, "executor")
+    cleanup_executor_cache(validation_executor_cache, "validation executor")
+    
+    return test_results

alita_sdk/configurations/openapi.py

@@ -64,7 +64,7 @@ class OpenApiConfiguration(BaseModel):
         ),
     )
     auth_type: Optional[Literal['Basic', 'Bearer', 'Custom']] = Field(
-        default='Bearer',
+        default=None,
         description=(
             "How to apply the API key. "
             "- 'Bearer': sets 'Authorization: Bearer <api_key>' "
@@ -111,7 +111,7 @@ class OpenApiConfiguration(BaseModel):
         )
     )
     method: Optional[Literal['default', 'Basic']] = Field(
-        default='default',
+        default=None,
         description=(
             "Token exchange method for client credentials flow. "
             "'default': Sends client_id and client_secret in POST body (Azure AD, Auth0, most providers). "

alita_sdk/runtime/clients/artifact.py

@@ -65,7 +65,7 @@ class Artifact:
     def delete(self, artifact_name: str, bucket_name = None):
         if not bucket_name:
             bucket_name = self.bucket_name
-        self.client.delete_artifact(bucket_name, artifact_name)
+        return self.client.delete_artifact(bucket_name, artifact_name)
     
     def list(self, bucket_name: str = None, return_as_string = True) -> str|dict:
         if not bucket_name:

alita_sdk/runtime/tools/artifact.py

@@ -21,13 +21,255 @@ from ...runtime.utils.utils import IndexerKeywords, resolve_image_from_cache
 class ArtifactWrapper(NonCodeIndexerToolkit):
     bucket: str
     artifact: Optional[Any] = None
-    
-    # Import file operation methods from BaseCodeToolApiWrapper
-    read_file_chunk = BaseCodeToolApiWrapper.read_file_chunk
-    read_multiple_files = BaseCodeToolApiWrapper.read_multiple_files
-    search_file = BaseCodeToolApiWrapper.search_file
-    edit_file = BaseCodeToolApiWrapper.edit_file
-    
+
+    # Override file operation methods to support bucket_name parameter
+    # (instead of importing from BaseCodeToolApiWrapper which uses 'branch')
+
+    def read_file_chunk(
+        self,
+        file_path: str,
+        start_line: int,
+        end_line: Optional[int] = None,
+        bucket_name: str = None
+    ) -> str:
+        """
+        Read a specific range of lines from a file in an artifact bucket.
+
+        Args:
+            file_path: Path to the file to read
+            start_line: Starting line number (1-indexed, inclusive)
+            end_line: Ending line number (1-indexed, inclusive). If None, reads to end.
+            bucket_name: Bucket name. If not provided, uses toolkit-configured default bucket.
+
+        Returns:
+            File content for the specified line range
+        """
+        from ...tools.utils.text_operations import apply_line_slice
+
+        # Calculate offset and limit from start_line and end_line
+        offset = start_line
+        limit = (end_line - start_line + 1) if end_line is not None else None
+
+        # Read the file with bucket_name support
+        content = self._read_file(file_path, branch=None, bucket_name=bucket_name, offset=offset, limit=limit)
+
+        # Apply client-side slicing if toolkit doesn't support partial reads
+        return apply_line_slice(content, offset=offset, limit=limit)
+
+    def read_multiple_files(
+        self,
+        file_paths: List[str],
+        bucket_name: str = None,
+        offset: Optional[int] = None,
+        limit: Optional[int] = None
+    ) -> dict:
+        """
+        Read multiple files in batch from an artifact bucket.
+
+        Args:
+            file_paths: List of file paths to read
+            bucket_name: Bucket name. If not provided, uses toolkit-configured default bucket.
+            offset: Starting line number for all files (1-indexed)
+            limit: Number of lines to read from offset for all files
+
+        Returns:
+            Dict mapping file paths to their content
+        """
+        from ...tools.utils.text_operations import apply_line_slice
+
+        results = {}
+        for path in file_paths:
+            try:
+                content = self._read_file(path, branch=None, bucket_name=bucket_name, offset=offset, limit=limit)
+                results[path] = apply_line_slice(content, offset=offset, limit=limit)
+            except Exception as e:
+                results[path] = f"Error reading file: {str(e)}"
+        return results
+
+    def search_file(
+        self,
+        file_path: str,
+        pattern: str,
+        bucket_name: str = None,
+        is_regex: bool = True,
+        context_lines: int = 2
+    ) -> str:
+        """
+        Search for a pattern in a file from an artifact bucket.
+
+        Args:
+            file_path: Path to the file to search
+            pattern: Search pattern. Treated as regex by default unless is_regex=False.
+            bucket_name: Bucket name. If not provided, uses toolkit-configured default bucket.
+            is_regex: Whether pattern is a regex. Default is True for flexible matching.
+            context_lines: Number of lines before/after match to include for context
+
+        Returns:
+            Formatted string with match results and context
+        """
+        from ...tools.utils.text_operations import search_in_content
+
+        content = self._read_file(file_path, branch=None, bucket_name=bucket_name)
+        matches = search_in_content(content, pattern, is_regex=is_regex, context_lines=context_lines)
+
+        if not matches:
+            return f"No matches found for pattern '{pattern}' in {file_path}"
+
+        # Format results
+        results = [f"Found {len(matches)} match(es) in {file_path}:\n"]
+        for match in matches:
+            results.append(f"\n--- Line {match['line_number']} ---")
+            if match['context_before']:
+                results.append("\n".join(f"  {l}" for l in match['context_before']))
+            results.append(f"> {match['line_content']}")
+            if match['context_after']:
+                results.append("\n".join(f"  {l}" for l in match['context_after']))
+
+        return "\n".join(results)
+
+    def edit_file(
+        self,
+        file_path: str,
+        file_query: str,
+        bucket_name: str = None,
+        commit_message: str = None
+    ) -> str:
+        """
+        Edit a file in an artifact bucket using OLD/NEW markers.
+
+        Args:
+            file_path: Path to the file to edit. Must be a text file.
+            file_query: Edit instructions with OLD/NEW markers.
+            bucket_name: Bucket name. If not provided, uses toolkit-configured default bucket.
+            commit_message: Not used for artifacts (kept for API consistency)
+
+        Returns:
+            Success message or error description
+        """
+        from ...tools.utils.text_operations import parse_old_new_markers, is_text_editable, try_apply_edit
+        from langchain_core.tools import ToolException
+
+        # Validate file type
+        if not is_text_editable(file_path):
+            raise ToolException(f"File '{file_path}' is not a text-editable file type")
+
+        # Read current content
+        content = self._read_file(file_path, branch=None, bucket_name=bucket_name)
+
+        # Parse edit instructions
+        edits = parse_old_new_markers(file_query)
+        if not edits:
+            raise ToolException("No valid OLD/NEW marker pairs found in edit instructions")
+
+        # Apply edits
+        updated_content = content
+        applied_count = 0
+        for old_text, new_text in edits:
+            updated_content, used_fallback = try_apply_edit(updated_content, old_text, new_text, file_path)
+            if updated_content != content or used_fallback:
+                applied_count += 1
+                content = updated_content
+
+        if applied_count == 0:
+            return f"No edits were applied to {file_path}. The OLD blocks may not match the file content."
+
+        # Write updated content
+        self._write_file(file_path, updated_content, branch=None, commit_message=commit_message, bucket_name=bucket_name)
+
+        return f"Successfully applied {applied_count} edit(s) to {file_path}"
+
+    def _get_file_operation_schemas(self):
+        """
+        Returns custom schemas for file operations that use bucket_name instead of branch.
+
+        This method is called by the @extend_with_file_operations decorator to get
+        toolkit-specific schemas for file operation tools.
+        """
+        # Artifact-specific schemas with bucket_name instead of branch
+        ArtifactReadFileChunkInput = create_model(
+            "ArtifactReadFileChunkInput",
+            file_path=(str, Field(description="Path to the file to read")),
+            bucket_name=(Optional[str], Field(
+                description="Bucket name. If not provided, uses toolkit-configured default bucket.",
+                default=None
+            )),
+            start_line=(int, Field(description="Starting line number (1-indexed, inclusive)", ge=1)),
+            end_line=(Optional[int], Field(
+                description="Ending line number (1-indexed, inclusive). If None, reads to end.",
+                default=None,
+                ge=1
+            )),
+        )
+
+        ArtifactReadMultipleFilesInput = create_model(
+            "ArtifactReadMultipleFilesInput",
+            file_paths=(List[str], Field(description="List of file paths to read", min_length=1)),
+            bucket_name=(Optional[str], Field(
+                description="Bucket name. If not provided, uses toolkit-configured default bucket.",
+                default=None
+            )),
+            offset=(Optional[int], Field(
+                description="Starting line number for all files (1-indexed)",
+                default=None,
+                ge=1
+            )),
+            limit=(Optional[int], Field(
+                description="Number of lines to read from offset for all files",
+                default=None,
+                ge=1
+            )),
+        )
+
+        ArtifactSearchFileInput = create_model(
+            "ArtifactSearchFileInput",
+            file_path=(str, Field(description="Path to the file to search")),
+            pattern=(str, Field(description="Search pattern. Treated as regex by default unless is_regex=False.")),
+            bucket_name=(Optional[str], Field(
+                description="Bucket name. If not provided, uses toolkit-configured default bucket.",
+                default=None
+            )),
+            is_regex=(bool, Field(
+                description="Whether pattern is a regex. Default is True for flexible matching.",
+                default=True
+            )),
+            context_lines=(int, Field(
+                description="Number of lines before/after match to include for context",
+                default=2,
+                ge=0
+            )),
+        )
+
+        ArtifactEditFileInput = create_model(
+            "ArtifactEditFileInput",
+            file_path=(str, Field(
+                description="Path to the file to edit. Must be a text file (markdown, txt, csv, json, xml, html, yaml, etc.)"
+            )),
+            file_query=(str, Field(description="""Edit instructions with OLD/NEW markers. Format:
+OLD <<<<
+old content to replace
+>>>> OLD
+NEW <<<<
+new content
+>>>> NEW
+
+Multiple OLD/NEW pairs can be provided for multiple edits.""")),
+            bucket_name=(Optional[str], Field(
+                description="Bucket name. If not provided, uses toolkit-configured default bucket.",
+                default=None
+            )),
+            commit_message=(Optional[str], Field(
+                description="Not used for artifacts (kept for API consistency)",
+                default=None
+            )),
+        )
+
+        return {
+            "read_file_chunk": ArtifactReadFileChunkInput,
+            "read_multiple_files": ArtifactReadMultipleFilesInput,
+            "search_file": ArtifactSearchFileInput,
+            "edit_file": ArtifactEditFileInput,
+        }
+
     @model_validator(mode='before')
     @classmethod
     def validate_toolkit(cls, values):
@@ -287,7 +529,10 @@ class ArtifactWrapper(NonCodeIndexerToolkit):
             raise ToolException(f"Unable to write file {file_path}: {str(e)}")
 
     def delete_file(self, filename: str, bucket_name = None):
-        return self.artifact.delete(filename, bucket_name)
+        result = self.artifact.delete(filename, bucket_name)
+        if result and isinstance(result, dict) and result.get('error'):
+            raise ToolException(f'Error (deleteFile): {result.get("error")}')
+        return f'File "{filename}" deleted successfully.'
 
     def append_data(self, filename: str, filedata: str, bucket_name = None):
         result = self.artifact.append(filename, filedata, bucket_name)

alita_sdk/runtime/tools/llm.py

@@ -1,7 +1,7 @@
 import asyncio
 import logging
 from traceback import format_exc
-from typing import Any, Optional, List, Union, Literal
+from typing import Any, Optional, List, Union, Literal, Dict
 
 from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
 from langchain_core.runnables import RunnableConfig
@@ -67,7 +67,7 @@ class LLMNode(BaseTool):
     client: Any = Field(default=None, description='LLM client instance')
     return_type: str = Field(default="str", description='Return type')
     response_key: str = Field(default="messages", description='Response key')
-    structured_output_dict: Optional[dict[str, str]] = Field(default=None, description='Structured output dictionary')
+    structured_output_dict: Optional[Dict[str, Any]] = Field(default=None, description='Structured output dictionary')
     output_variables: Optional[List[str]] = Field(default=None, description='Output variables')
     input_mapping: Optional[dict[str, dict]] = Field(default=None, description='Input mapping')
     input_variables: Optional[List[str]] = Field(default=None, description='Input variables')
@@ -82,7 +82,7 @@ class LLMNode(BaseTool):
         Prepare structured output parameters from structured_output_dict.
 
         Expected self.structured_output_dict formats:
-          - {"field": "str"} / {"field": "list"} / {"field": "list[str]"} / {"field": "any"} ...
+          - {"field": "str"} / {"field": "list"} / {"field": "list[dict]"} / {"field": "any"} ...
           - OR {"field": {"type": "...", "description": "...", "default": ...}}  (optional)
 
         Returns:
@@ -93,19 +93,20 @@ class LLMNode(BaseTool):
         for key, value in (self.structured_output_dict or {}).items():
             # Allow either a plain type string or a dict with details
             if isinstance(value, dict):
-                type_str = (value.get("type") or "any")
+                type_str = str(value.get("type") or "any")
                 desc = value.get("description", "") or ""
                 entry: dict = {"type": type_str, "description": desc}
                 if "default" in value:
                     entry["default"] = value["default"]
             else:
-                type_str = (value or "any") if isinstance(value, str) else "any"
-                entry = {"type": type_str, "description": ""}
+                # Ensure we always have a string type
+                if isinstance(value, str):
+                    type_str = value
+                else:
+                    # If it's already a type object, convert to string representation
+                    type_str = getattr(value, '__name__', 'any')
 
-            # Normalize: only convert the *exact* "list" into "list[str]"
-            # (avoid the old bug where "if 'list' in value" also hits "blacklist", etc.)
-            if isinstance(entry.get("type"), str) and entry["type"].strip().lower() == "list":
-                entry["type"] = "list[str]"
+                entry = {"type": type_str, "description": ""}
 
             struct_params[key] = entry
 
@@ -1146,5 +1147,5 @@ class LLMNode(BaseTool):
 
         return new_messages, current_completion
 
-    def __get_struct_output_model(self, llm_client, pydantic_model, method: Literal["function_calling", "json_mode", "json_schema"] = "json_schema"):
+    def __get_struct_output_model(self, llm_client, pydantic_model, method: Literal["function_calling", "json_mode", "json_schema"] = "function_calling"):
         return llm_client.with_structured_output(pydantic_model, method=method)