alita-sdk 0.3.263__py3-none-any.whl → 0.3.499__py3-none-any.whl

alita_sdk/runtime/tools/application.py

@@ -1,9 +1,9 @@
 import json
 
 from ..utils.utils import clean_string
-from langchain_core.tools import BaseTool
-from langchain_core.messages import BaseMessage, AIMessage, ToolCall
-from typing import Any, Type, Optional, Union
+from langchain_core.tools import BaseTool, ToolException
+from langchain_core.messages import BaseMessage, AIMessage, HumanMessage
+from typing import Any, Type, Optional
 from pydantic import create_model, field_validator, BaseModel
 from pydantic.fields import FieldInfo
 from ..langchain.mixedAgentRenderes import convert_message_to_json
@@ -31,7 +31,12 @@ def formulate_query(kwargs):
             chat_history = []
             for each in kwargs.get('chat_history')[:]:
                 chat_history.append(AIMessage(each))
-    result = {"input": kwargs.get('task'), "chat_history": chat_history}
+    user_task = kwargs.get('task')
+    if not user_task:
+        raise ToolException("Task is required to invoke the application. "
+                            "Check the provided input (some errors may happen on previous steps).")
+    input_message = HumanMessage(content=user_task)
+    result = {"input": [input_message], "chat_history": chat_history}
     for key, value in kwargs.items():
         if key not in ("task", "chat_history"):
             result[key] = value
@@ -45,6 +50,8 @@ class Application(BaseTool):
     application: Any
     args_schema: Type[BaseModel] = applicationToolSchema
     return_type: str = "str"
+    client: Any
+    args_runnable: dict = {}
     
     @field_validator('name', mode='before')
     @classmethod
@@ -61,6 +68,11 @@ class Application(BaseTool):
         return self._run(*config, **all_kwargs)
 
     def _run(self, *args, **kwargs):
+        if self.client and self.args_runnable:
+            # Recreate new LanggraphAgentRunnable in order to reflect the current input_mapping (it can be dynamic for pipelines).
+            # Actually, for pipelines agent toolkits LanggraphAgentRunnable is created (for LLMNode) before pipeline's schema parsing.
+            application_variables = {k: {"name": k, "value": v} for k, v in kwargs.items()}
+            self.application = self.client.application(**self.args_runnable, application_variables=application_variables)
         response = self.application.invoke(formulate_query(kwargs))
         if self.return_type == "str":
             return response["output"]

alita_sdk/runtime/tools/artifact.py

@@ -1,14 +1,20 @@
 import hashlib
+import io
 import json
 import logging
-from typing import Any, Optional, Generator
+import re
+from typing import Any, Optional, Generator, List
 
+from langchain_core.callbacks import dispatch_custom_event
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
+from openpyxl.workbook.workbook import Workbook
 from pydantic import create_model, Field, model_validator
 
-from alita_sdk.tools.non_code_indexer_toolkit import NonCodeIndexerToolkit
-from alita_sdk.tools.utils.available_tools_decorator import extend_with_parent_available_tools
+from ...tools.non_code_indexer_toolkit import NonCodeIndexerToolkit
+from ...tools.utils.available_tools_decorator import extend_with_parent_available_tools
+from ...tools.elitea_base import extend_with_file_operations
+from ...runtime.utils.utils import IndexerKeywords
 
 
 class ArtifactWrapper(NonCodeIndexerToolkit):
@@ -26,10 +32,105 @@ class ArtifactWrapper(NonCodeIndexerToolkit):
         return super().validate_toolkit(values)
 
     def list_files(self, bucket_name = None, return_as_string = True):
-        return self.artifact.list(bucket_name, return_as_string)
+        """List all files in the artifact bucket with API download links."""
+        result = self.artifact.list(bucket_name, return_as_string=False)
+        
+        # Add API download link to each file
+        if isinstance(result, dict) and 'rows' in result:
+            bucket = bucket_name or self.bucket
+            
+            # Get base_url and project_id from alita client
+            base_url = getattr(self.alita, 'base_url', '').rstrip('/')
+            project_id = getattr(self.alita, 'project_id', '')
+            
+            for file_info in result['rows']:
+                if 'name' in file_info:
+                    # Generate API download link
+                    file_name = file_info['name']
+                    file_info['link'] = f"{base_url}/api/v2/artifacts/artifact/default/{project_id}/{bucket}/{file_name}"
+        
+        return str(result) if return_as_string else result
 
     def create_file(self, filename: str, filedata: str, bucket_name = None):
-        return self.artifact.create(filename, filedata, bucket_name)
+        # Sanitize filename to prevent regex errors during indexing
+        sanitized_filename, was_modified = self._sanitize_filename(filename)
+        if was_modified:
+            logging.warning(f"Filename sanitized: '{filename}' -> '{sanitized_filename}'")
+        
+        if sanitized_filename.endswith(".xlsx"):
+            data = json.loads(filedata)
+            filedata = self.create_xlsx_filedata(data)
+
+        result = self.artifact.create(sanitized_filename, filedata, bucket_name)
+        
+        # Dispatch custom event for file creation
+        dispatch_custom_event("file_modified", {
+            "message": f"File '{filename}' created successfully",
+            "filename": filename,
+            "tool_name": "createFile",
+            "toolkit": "artifact",
+            "operation_type": "create",
+            "meta": {
+                "bucket": bucket_name or self.bucket
+            }
+        })
+
+        return result
+    
+    @staticmethod
+    def _sanitize_filename(filename: str) -> tuple:
+        """Sanitize filename for safe storage and regex pattern matching."""
+        from pathlib import Path
+        
+        if not filename or not filename.strip():
+            return "unnamed_file", True
+        
+        original = filename
+        path_obj = Path(filename)
+        name = path_obj.stem
+        extension = path_obj.suffix
+        
+        # Whitelist: alphanumeric, underscore, hyphen, space, Unicode letters/digits
+        sanitized_name = re.sub(r'[^\w\s-]', '', name, flags=re.UNICODE)
+        sanitized_name = re.sub(r'[-\s]+', '-', sanitized_name)
+        sanitized_name = sanitized_name.strip('-').strip()
+        
+        if not sanitized_name:
+            sanitized_name = "file"
+        
+        if extension:
+            extension = re.sub(r'[^\w.-]', '', extension, flags=re.UNICODE)
+        
+        sanitized = sanitized_name + extension
+        return sanitized, (sanitized != original)
+
+    def create_xlsx_filedata(self, data: dict[str, list[list]]) -> bytes:
+        try:
+            workbook = Workbook()
+
+            first_sheet = True
+            for sheet_name, sheet_data in data.items():
+                if first_sheet:
+                    sheet = workbook.active
+                    sheet.title = sheet_name
+                    first_sheet = False
+                else:
+                    sheet = workbook.create_sheet(title=sheet_name)
+
+                for row in sheet_data:
+                    sheet.append(row)
+
+            file_buffer = io.BytesIO()
+            workbook.save(file_buffer)
+            file_buffer.seek(0)
+
+            return file_buffer.read()
+
+        except json.JSONDecodeError:
+            raise ValueError("Invalid JSON format for .xlsx file data.")
+        except Exception as e:
+            raise ValueError(f"Error processing .xlsx file data: {e}")
+
 
     def read_file(self,
                   filename: str,
@@ -45,31 +146,187 @@ class ArtifactWrapper(NonCodeIndexerToolkit):
                                   sheet_name=sheet_name,
                                   excel_by_sheets=excel_by_sheets,
                                   llm=self.llm)
+    
+    def _read_file(
+        self,
+        file_path: str,
+        branch: str = None,
+        bucket_name: str = None,
+        **kwargs
+    ) -> str:
+        """
+        Read a file from artifact bucket with optional partial read support.
+        
+        Parameters:
+            file_path: Name of the file in the bucket
+            branch: Not used for artifacts (kept for API consistency)
+            bucket_name: Name of the bucket (uses default if None)
+            **kwargs: Additional parameters (offset, limit, head, tail) - currently ignored,
+                     partial read handled client-side by base class methods
+        
+        Returns:
+            File content as string
+        """
+        return self.read_file(filename=file_path, bucket_name=bucket_name)
+    
+    def _write_file(
+        self,
+        file_path: str,
+        content: str,
+        branch: str = None,
+        commit_message: str = None,
+        bucket_name: str = None
+    ) -> str:
+        """
+        Write content to a file (create or overwrite).
+        
+        Parameters:
+            file_path: Name of the file in the bucket
+            content: New file content
+            branch: Not used for artifacts (kept for API consistency)
+            commit_message: Not used for artifacts (kept for API consistency)
+            bucket_name: Name of the bucket (uses default if None)
+            
+        Returns:
+            Success message
+        """
+        try:
+            # Sanitize filename
+            sanitized_filename, was_modified = self._sanitize_filename(file_path)
+            if was_modified:
+                logging.warning(f"Filename sanitized: '{file_path}' -> '{sanitized_filename}'")
+            
+            # Check if file exists
+            try:
+                self.artifact.get(artifact_name=sanitized_filename, bucket_name=bucket_name, llm=self.llm)
+                # File exists, overwrite it
+                result = self.artifact.overwrite(sanitized_filename, content, bucket_name)
+                
+                # Dispatch custom event
+                dispatch_custom_event("file_modified", {
+                    "message": f"File '{sanitized_filename}' updated successfully",
+                    "filename": sanitized_filename,
+                    "tool_name": "edit_file",
+                    "toolkit": "artifact",
+                    "operation_type": "modify",
+                    "meta": {
+                        "bucket": bucket_name or self.bucket
+                    }
+                })
+                
+                return f"Updated file {sanitized_filename}"
+            except:
+                # File doesn't exist, create it
+                result = self.artifact.create(sanitized_filename, content, bucket_name)
+                
+                # Dispatch custom event
+                dispatch_custom_event("file_modified", {
+                    "message": f"File '{sanitized_filename}' created successfully",
+                    "filename": sanitized_filename,
+                    "tool_name": "edit_file",
+                    "toolkit": "artifact",
+                    "operation_type": "create",
+                    "meta": {
+                        "bucket": bucket_name or self.bucket
+                    }
+                })
+                
+                return f"Created file {sanitized_filename}"
+        except Exception as e:
+            raise ToolException(f"Unable to write file {file_path}: {str(e)}")
 
     def delete_file(self, filename: str, bucket_name = None):
         return self.artifact.delete(filename, bucket_name)
 
     def append_data(self, filename: str, filedata: str, bucket_name = None):
-        return self.artifact.append(filename, filedata, bucket_name)
+        result = self.artifact.append(filename, filedata, bucket_name)
+        
+        # Dispatch custom event for file append
+        dispatch_custom_event("file_modified", {
+            "message": f"Data appended to file '{filename}' successfully",
+            "filename": filename,
+            "tool_name": "appendData",
+            "toolkit": "artifact",
+            "operation_type": "modify",
+            "meta": {
+                "bucket": bucket_name or self.bucket
+            }
+        })
+        
+        return result
 
     def overwrite_data(self, filename: str, filedata: str, bucket_name = None):
-        return self.artifact.overwrite(filename, filedata, bucket_name)
+        result = self.artifact.overwrite(filename, filedata, bucket_name)
+        
+        # Dispatch custom event for file overwrite
+        dispatch_custom_event("file_modified", {
+            "message": f"File '{filename}' overwritten successfully",
+            "filename": filename,
+            "tool_name": "overwriteData",
+            "toolkit": "artifact",
+            "operation_type": "modify",
+            "meta": {
+                "bucket": bucket_name or self.bucket
+            }
+        })
+        
+        return result
 
     def create_new_bucket(self, bucket_name: str, expiration_measure = "weeks", expiration_value = 1):
-        return self.artifact.client.create_bucket(bucket_name, expiration_measure, expiration_value)
+        # Sanitize bucket name: replace underscores with hyphens and ensure lowercase
+        sanitized_name = bucket_name.replace('_', '-').lower()
+        if sanitized_name != bucket_name:
+            logging.warning(f"Bucket name '{bucket_name}' was sanitized to '{sanitized_name}' (underscores replaced with hyphens, converted to lowercase)")
+        return self.artifact.client.create_bucket(sanitized_name, expiration_measure, expiration_value)
+
+    def _index_tool_params(self):
+        return {
+            'include_extensions': (Optional[List[str]], Field(
+                description="List of file extensions to include when processing: i.e. ['*.png', '*.jpg']. "
+                            "If empty, all files will be processed (except skip_extensions).",
+                default=[])),
+            'skip_extensions': (Optional[List[str]], Field(
+                description="List of file extensions to skip when processing: i.e. ['*.png', '*.jpg']",
+                default=[])),
+        }
 
     def _base_loader(self, **kwargs) -> Generator[Document, None, None]:
+        self._log_tool_event(message=f"Loading the files from artifact's bucket. {kwargs=}", tool_name="loader")
         try:
-            all_files = self.list_files(self.bucket, False)
+            all_files = self.list_files(self.bucket, False)['rows']
         except Exception as e:
             raise ToolException(f"Unable to extract files: {e}")
 
-        for file in all_files['rows']:
+        include_extensions = kwargs.get('include_extensions', [])
+        skip_extensions = kwargs.get('skip_extensions', [])
+        self._log_tool_event(message=f"Files filtering started. Include extensions: {include_extensions}. "
+                                     f"Skip extensions: {skip_extensions}", tool_name="loader")
+        # show the progress of filtering
+        total_files = len(all_files) if isinstance(all_files, list) else 0
+        filtered_files_count = 0
+        for file in all_files:
+            filtered_files_count += 1
+            if filtered_files_count % 10 == 0 or filtered_files_count == total_files:
+                self._log_tool_event(message=f"Files filtering progress: {filtered_files_count}/{total_files}",
+                                     tool_name="loader")
+            file_name = file['name']
+
+            # Check if file should be skipped based on skip_extensions
+            if any(re.match(re.escape(pattern).replace(r'\*', '.*') + '$', file_name, re.IGNORECASE)
+                   for pattern in skip_extensions):
+                continue
+
+            # Check if file should be included based on include_extensions
+            # If include_extensions is empty, process all files (that weren't skipped)
+            if include_extensions and not (any(re.match(re.escape(pattern).replace(r'\*', '.*') + '$', file_name, re.IGNORECASE)
+                                               for pattern in include_extensions)):
+                continue
+
             metadata = {
                 ("updated_on" if k == "modified" else k): str(v)
                 for k, v in file.items()
             }
-            metadata['id'] = self.get_hash_from_bucket_and_file_name(self.bucket, file['name'])
+            metadata['id'] = self.get_hash_from_bucket_and_file_name(self.bucket, file_name)
             yield Document(page_content="", metadata=metadata)
 
     def get_hash_from_bucket_and_file_name(self, bucket, file_name):
@@ -82,21 +339,24 @@ class ArtifactWrapper(NonCodeIndexerToolkit):
         for document in documents:
             try:
                 page_content = self.artifact.get_content_bytes(artifact_name=document.metadata['name'])
-                document.metadata['loader_content'] = page_content
-                document.metadata['loader_content_type'] = document.metadata['name']
+                document.metadata[IndexerKeywords.CONTENT_IN_BYTES.value] = page_content
+                document.metadata[IndexerKeywords.CONTENT_FILE_NAME.value] = document.metadata['name']
                 yield document
             except Exception as e:
-                logging.error(f"Failed while parsing the file '{document.metadata['name']}': {e}")
+                logger.error(f"Failed while parsing the file '{document.metadata['name']}': {e}")
                 yield document
 
-    @extend_with_parent_available_tools
+    @extend_with_file_operations
     def get_available_tools(self):
+        """Get available tools, including indexing tools only if vector store is configured."""
         bucket_name = (Optional[str], Field(description="Name of the bucket to work with."
                                                         "If bucket is not specified by user directly, the name should be taken from chat history."
                                                         "If bucket never mentioned in chat, the name will be taken from tool configuration."
                                                         " ***IMPORTANT*** Underscore `_` is prohibited in bucket name and should be replaced by `-`",
                                             default=None))
-        return [
+        
+        # Basic artifact tools (always available)
+        basic_tools = [
             {
                 "ref": self.list_files,
                 "name": "listFiles",
@@ -110,7 +370,17 @@ class ArtifactWrapper(NonCodeIndexerToolkit):
                 "args_schema": create_model(
                     "createFile", 
                     filename=(str, Field(description="Filename")),
-                    filedata=(str, Field(description="Stringified content of the file")),
+                    filedata=(str, Field(description="""Stringified content of the file.
+                    Example for .xlsx filedata format:
+                    {
+                        "Sheet1":[
+                            ["Name", "Age", "City"],
+                            ["Alice", 25, "New York"],
+                            ["Bob", 30, "San Francisco"],
+                            ["Charlie", 35, "Los Angeles"]
+                        ]
+                    }
+                    """)),
                     bucket_name=bucket_name
                 )
             },
@@ -171,11 +441,34 @@ class ArtifactWrapper(NonCodeIndexerToolkit):
                 "description": "Creates new bucket specified by user.",
                 "args_schema": create_model(
                     "createNewBucket",
-                    bucket_name=(str, Field(description="Bucket name to create. ***IMPORTANT*** Underscore `_` is prohibited in bucket name and should be replaced by `-`.")),
+                    bucket_name=(str, Field(
+                        description="Bucket name to create. Must start with lowercase letter and contain only lowercase letters, numbers, and hyphens. Underscores will be automatically converted to hyphens.",
+                        pattern=r'^[a-z][a-z0-9_-]*$'  # Allow underscores in input, will be sanitized
+                    )),
                     expiration_measure=(Optional[str], Field(description="Measure of expiration time for bucket configuration."
                                                                          "Possible values: `days`, `weeks`, `months`, `years`.",
                                                              default="weeks")),
                     expiration_value=(Optional[int], Field(description="Expiration time values.", default=1))
                 )
             }
-        ]
+        ]
+        
+        # Add indexing tools only if vector store is configured
+        has_vector_config = (
+            hasattr(self, 'embedding_model') and self.embedding_model and
+            hasattr(self, 'pgvector_configuration') and self.pgvector_configuration
+        )
+        
+        if has_vector_config:
+            try:
+                # Get indexing tools from parent class
+                indexing_tools = super(ArtifactWrapper, self).get_available_tools()
+                return indexing_tools + basic_tools
+            except Exception as e:
+                # If getting parent tools fails, log warning and return basic tools only
+                logging.warning(f"Failed to load indexing tools: {e}. Only basic artifact tools will be available.")
+                return basic_tools
+        else:
+            # No vector store config, return basic tools only
+            logging.info("Vector store not configured. Indexing tools (index_data, search_index, etc.) are not available.")
+            return basic_tools

alita_sdk/runtime/tools/function.py

@@ -1,18 +1,33 @@
+import json
 import logging
+from copy import deepcopy
 from json import dumps
 
 from langchain_core.callbacks import dispatch_custom_event
 from langchain_core.messages import ToolCall
 from langchain_core.runnables import RunnableConfig
-from langchain_core.tools import BaseTool
-from typing import Any, Optional, Union, Annotated
+from langchain_core.tools import BaseTool, ToolException
+from typing import Any, Optional, Union
 from langchain_core.utils.function_calling import convert_to_openai_tool
 from pydantic import ValidationError
+
 from ..langchain.utils import propagate_the_input_mapping
 
 logger = logging.getLogger(__name__)
 
 
+def replace_escaped_newlines(data):
+    """
+        Replace \\n with \n in all string values recursively.
+        Required for sanitization of state variables in code node
+    """
+    if isinstance(data, dict):
+        return {key: replace_escaped_newlines(value) for key, value in data.items()}
+    elif isinstance(data, str):
+        return data.replace('\\n', '\n')
+    else:
+        return data
+
 class FunctionTool(BaseTool):
     name: str = 'FunctionalTool'
     description: str = 'This is direct call node for tools'
@@ -21,6 +36,61 @@ class FunctionTool(BaseTool):
     input_variables: Optional[list[str]] = None
     input_mapping: Optional[dict[str, dict]] = None
     output_variables: Optional[list[str]] = None
+    structured_output: Optional[bool] = False
+    alita_client: Optional[Any] = None
+
+    def _prepare_pyodide_input(self, state: Union[str, dict, ToolCall]) -> str:
+        """Prepare input for PyodideSandboxTool by injecting state into the code block."""
+        # add state into the code block here since it might be changed during the execution of the code
+        state_copy = replace_escaped_newlines(deepcopy(state))
+
+        del state_copy['messages']  # remove messages to avoid issues with pickling without langchain-core
+        # inject state into the code block as alita_state variable
+        state_json = json.dumps(state_copy, ensure_ascii=False)
+        pyodide_predata = f'#state dict\nimport json\nalita_state = json.loads({json.dumps(state_json)})\n'
+
+        return pyodide_predata
+
+    def _handle_pyodide_output(self, tool_result: Any) -> dict:
+        """Handle output processing for PyodideSandboxTool results."""
+        tool_result_converted = {}
+
+        if self.output_variables:
+            for var in self.output_variables:
+                if var == "messages":
+                    tool_result_converted.update(
+                        {"messages": [{"role": "assistant", "content": dumps(tool_result)}]})
+                    continue
+                if isinstance(tool_result, dict) and var in tool_result:
+                    tool_result_converted[var] = tool_result[var]
+                else:
+                    # handler in case user points to a var that is not in the output of the tool
+                    tool_result_converted[var] = tool_result.get('result',
+                                                                 tool_result.get('error') if tool_result.get('error')
+                                                                 else 'Execution result is missing')
+        else:
+            tool_result_converted.update({"messages": [{"role": "assistant", "content": dumps(tool_result)}]})
+
+        if self.structured_output:
+            # execute code tool and update state variables
+            try:
+                result_value = tool_result.get('result', {})
+                if isinstance(result_value, dict):
+                    tool_result_converted.update(result_value)
+                elif isinstance(result_value, list):
+                    # Handle list case - could wrap in a key or handle differently based on requirements
+                    tool_result_converted.update({"result": result_value})
+                else:
+                    # Handle JSON string case
+                    tool_result_converted.update(json.loads(result_value))
+            except json.JSONDecodeError:
+                logger.error(f"JSONDecodeError: {tool_result}")
+
+        return tool_result_converted
+
+    def _is_pyodide_tool(self) -> bool:
+        """Check if the current tool is a PyodideSandboxTool."""
+        return self.tool.name.lower() == 'pyodide_sandbox'
 
     def invoke(
             self,
@@ -31,8 +101,15 @@ class FunctionTool(BaseTool):
         params = convert_to_openai_tool(self.tool).get(
             'function', {'parameters': {}}).get(
             'parameters', {'properties': {}}).get('properties', {})
+
         func_args = propagate_the_input_mapping(input_mapping=self.input_mapping, input_variables=self.input_variables,
                                                 state=state)
+
+        # special handler for PyodideSandboxTool
+        if self._is_pyodide_tool():
+            # replace new lines in strings in code block
+            code = func_args['code'].replace('\\n', '\\\\n')
+            func_args['code'] = f"{self._prepare_pyodide_input(state)}\n{code}"
         try:
             tool_result = self.tool.invoke(func_args, config, **kwargs)
             dispatch_custom_event(
@@ -44,11 +121,30 @@ class FunctionTool(BaseTool):
                 }, config=config
             )
             logger.info(f"ToolNode response: {tool_result}")
+
+            # handler for PyodideSandboxTool
+            if self._is_pyodide_tool():
+                return self._handle_pyodide_output(tool_result)
+
             if not self.output_variables:
                 return {"messages": [{"role": "assistant", "content": dumps(tool_result)}]}
             else:
-                if self.output_variables[0] == "messages":
-                    return {"messages": [{"role": "assistant", "content": dumps(tool_result)}]}
+                if "messages" in self.output_variables:
+                    messages_dict = {
+                        "messages": [{
+                            "role": "assistant",
+                            "content": dumps(tool_result)
+                            if not isinstance(tool_result, ToolException) and not isinstance(tool_result, str)
+                            else str(tool_result)
+                        }]
+                    }
+                    for var in self.output_variables:
+                        if var != "messages":
+                            if isinstance(tool_result, dict) and var in tool_result:
+                                messages_dict[var] = tool_result[var]
+                            else:
+                                messages_dict[var] = tool_result
+                    return messages_dict
                 else:
                     return { self.output_variables[0]: tool_result }
         except ValidationError: