dhisana 0.0.1.dev243__py3-none-any.whl

dhisana/utils/openai_helpers.py

@@ -0,0 +1,977 @@
+# Helper functions to call OpenAI Assistant
+
+from datetime import datetime, timezone
+import inspect
+import os
+import csv
+import json
+import hashlib
+import asyncio
+import logging
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type
+
+from pydantic import BaseModel, Field, create_model
+from fastapi import HTTPException
+from openai import AsyncOpenAI, OpenAIError, pydantic_function_tool
+
+from dhisana.utils import cache_output_tools
+# from dhisana.utils.trasform_json import GLOBAL_GENERATED_PYTHON_CODE
+
+from .agent_tools import GLOBAL_DATA_MODELS, GLOBAL_TOOLS_FUNCTIONS
+from .google_workspace_tools import get_file_content_from_googledrive_by_name, write_content_to_googledrive
+from .agent_tools import GLOBAL_OPENAI_ASSISTANT_TOOLS
+from .openapi_spec_to_tools import (
+    OPENAPI_TOOL_CONFIGURATIONS,
+    OPENAPI_GLOBAL_OPENAI_ASSISTANT_TOOLS,
+    OPENAPI_CALLABALE_FUNCTIONS,
+)
+
+# This file has functions to execute the agent workflow using provided spec and OpenAPI.
+# Also has helper functions to extract structured data from the agent response.
+# TODO: we need to enhance the Agent workflow handling. 
+# TODO: Move the OpenAI related helper functions to a separate file.
+
+def get_openai_access_token(tool_config: Optional[List[Dict]] = None) -> str:
+    """
+    Retrieves the OPENAI_API_KEY access token from the provided tool configuration.
+
+    Args:
+        tool_config (list): A list of dictionaries containing the tool configuration. 
+                            Each dictionary should have a "name" key and a "configuration" key,
+                            where "configuration" is a list of dictionaries containing "name" and "value" keys.
+
+    Returns:
+        str: The OPENAI_API_KEY access token.
+
+    Raises:
+        ValueError: If the OpenAI integration has not been configured.
+    """
+    if tool_config:
+        openai_config = next(
+            (item for item in tool_config if item.get("name") == "openai"), None
+        )
+        if openai_config:
+            config_map = {
+                item["name"]: item["value"]
+                for item in openai_config.get("configuration", [])
+                if item
+            }
+            OPENAI_API_KEY = config_map.get("apiKey")
+        else:
+            OPENAI_API_KEY = None
+    else:
+        OPENAI_API_KEY = None
+
+    OPENAI_API_KEY = OPENAI_API_KEY or os.getenv("OPENAI_API_KEY")
+    if not OPENAI_API_KEY:
+        raise ValueError(
+            "OpenAI integration is not configured. Please configure the connection to OpenAI in Integrations."
+        )
+    return OPENAI_API_KEY
+
+async def read_from_google_drive(path):
+    return await get_file_content_from_googledrive_by_name(file_name=path)
+
+# Function to get headers for OpenAPI tools
+def get_headers(toolname):
+    headers = OPENAPI_TOOL_CONFIGURATIONS.get(toolname, {}).get("headers", {})
+    return headers
+
+
+def get_params(toolname):
+    params = OPENAPI_TOOL_CONFIGURATIONS.get(toolname, {}).get("params", {})
+    return params
+
+
+async def run_assistant(client, assistant, thread, prompt, response_type, allowed_tools):
+    """
+    Runs the assistant with the given parameters.
+    """
+    await send_initial_message(client, thread, prompt)
+    allowed_tool_items = get_allowed_tool_items(allowed_tools)
+    response_format = get_response_format(response_type)
+
+    max_iterations = 5
+    iteration_count = 0
+
+    while iteration_count < max_iterations:
+        run = await client.beta.threads.runs.create_and_poll(
+            thread_id=thread.id,
+            assistant_id=assistant.id,
+            response_format=response_format,
+            tools=allowed_tool_items,
+        )
+
+        while run.status == 'requires_action':
+            if iteration_count >= max_iterations:
+                logging.info("Exceeded maximum number of iterations for requires_action.")
+                await client.beta.threads.runs.cancel(run_id=run.id, thread_id=thread.id)
+                return "FAIL"
+
+            tool_outputs = await handle_required_action(run)
+            if tool_outputs:
+                run = await submit_tool_outputs(client, thread, run, tool_outputs)
+            else:
+                break
+            iteration_count += 1
+            logging.info("Iteration count: %s", iteration_count)
+
+        if run.status == 'completed':
+            status = await handle_run_completion(client, thread, run)
+            return status
+        elif run.status == 'failed' and run.last_error.code == 'rate_limit_exceeded':
+            logging.info("Rate limit exceeded. Retrying in 30 seconds...")
+            await asyncio.sleep(30)
+        elif run.status == 'expired':
+            logging.info("Run expired. Creating a new run...")
+        else:
+            logging.info(f"Run status: {run.status}")
+            return run.status
+
+        iteration_count += 1
+        if (iteration_count >= max_iterations):
+            logging.info("Exceeded maximum number of iterations.")
+            await client.beta.threads.runs.cancel(run_id=run.id, thread_id=thread.id)
+            return 'FAIL'
+        logging.info("Iteration count: %s", iteration_count)
+    
+    return "FAIL"
+
+async def handle_run_completion(client, thread, run):
+    if run.status == 'completed':
+        messages = await client.beta.threads.messages.list(thread_id=thread.id)
+        return messages.data[0].content[0].text.value
+    else:
+        return run.status
+
+
+async def send_initial_message(client, thread, prompt):
+    await client.beta.threads.messages.create(
+        thread_id=thread.id,
+        role="user",
+        content=prompt,
+    )
+
+
+def get_allowed_tool_items(allowed_tools):
+    allowed_tool_items = [
+        tool for tool in GLOBAL_OPENAI_ASSISTANT_TOOLS
+        if tool['type'] == 'function' and tool['function']['name'] in allowed_tools
+    ]
+    allowed_tool_items.extend([
+        tool for tool in OPENAPI_GLOBAL_OPENAI_ASSISTANT_TOOLS
+        if tool['type'] == 'function' and tool['function']['name'] in allowed_tools
+    ])
+    return allowed_tool_items
+
+
+def get_response_format(response_type):
+    return {
+        'type': 'json_schema',
+        'json_schema': {
+            "name": response_type.__name__,
+            "schema": response_type.model_json_schema()
+        }
+    }
+
+
+async def handle_required_action(run):
+    tool_outputs = []
+    current_batch_size = 0
+    max_batch_size = 256 * 1024  # 256 KB
+    logging.info(f"Handling required action")
+
+    if hasattr(run, 'required_action') and hasattr(run.required_action, 'submit_tool_outputs'):
+        for tool in run.required_action.submit_tool_outputs.tool_calls:
+            function, openai_function = get_function(tool.function.name)
+            if function:
+                output_str, output_size = await invoke_function(function, tool, openai_function)
+                if current_batch_size + output_size > max_batch_size:
+                    tool_outputs.append(
+                        {"tool_call_id": tool.id, "output": ""})
+                else:
+                    tool_outputs.append(
+                        {"tool_call_id": tool.id, "output": output_str})
+                    current_batch_size += output_size
+            else:
+                logging.info(f"Function {tool.function.name} not found.")
+                tool_outputs.append(
+                    {"tool_call_id": tool.id, "output": "No results found"})
+
+    return tool_outputs
+
+
+def get_function(function_name):
+    function = GLOBAL_TOOLS_FUNCTIONS.get(function_name)
+    openai_function = False
+    if not function:
+        function = OPENAPI_CALLABALE_FUNCTIONS.get(function_name)
+        openai_function = True
+    return function, openai_function
+
+
+async def invoke_function(function, tool, openai_function):
+    try:
+        function_args = json.loads(tool.function.arguments)
+        logging.info(f"Invoking function {tool.function.name} with args: {function_args}\n")
+              
+        if openai_function:
+            output = await invoke_openapi_function(function, function_args, tool.function.name)
+        else:
+            if asyncio.iscoroutinefunction(function):
+                output = await function(**function_args)
+            else:
+                output = function(**function_args)
+        output_str = json.dumps(output)
+        output_size = len(output_str.encode('utf-8'))
+        logging.info(f"\nOutput from function {tool.function.name}: {output_str[:256]}\n")
+              
+        return output_str, output_size
+    except Exception as e:
+        logging.info(f"invoke_function Error invoking function {tool.function.name}: {e}")
+        return "No results found", 0
+
+
+async def invoke_openapi_function(function, function_args, function_name):
+
+    json_body = function_args.get("json", None)
+    path_params = function_args.get("path_params", None)
+    fn_args = {"path_params": path_params, "data": json_body}
+    headers = get_headers(function_name)
+
+    query_params = function_args.get("params", {})
+    params = get_params(function_name)
+    query_params.update(params)
+    status, reason, text = await function(
+        name=function_name,
+        fn_args=fn_args,
+        headers=headers,
+        params=query_params,
+    )
+    logging.info(f"\nOutput from function {function_name}: {status} {reason}\n")
+    return {
+        "status_code": status,
+        "text": text,
+        "reason": reason,
+    }
+
+
+async def submit_tool_outputs(client, thread, run, tool_outputs):
+    try:
+        return await client.beta.threads.runs.submit_tool_outputs_and_poll(
+            thread_id=thread.id,
+            run_id=run.id,
+            tool_outputs=tool_outputs
+        )
+    except Exception as e:
+        logging.info(f"Failed to submit tool outputs: ${e}")
+        return run
+
+
+async def handle_run_completion(client, thread, run):
+    if run.status == 'completed':
+        messages = await client.beta.threads.messages.list(thread_id=thread.id)
+        return messages.data[0].content[0].text.value
+    else:
+        logging.info(f"Run status: {run.status}")
+        return run.status
+
+
+
+async def extract_and_structure_data(client, assistant, thread, prompt, task_inputs, response_type, allowed_tools):
+    # Replace placeholders in the prompt with task inputs
+    formatted_prompt = prompt
+    for key, value in task_inputs.items():
+        placeholder = "{{ inputs." + key + " }}"
+        formatted_prompt = formatted_prompt.replace(placeholder, str(value))
+    
+    # Create a hash of the formatted prompt
+    prompt_hash = hashlib.md5(formatted_prompt.encode()).hexdigest()
+    
+    # Retrieve cached response if available
+    cached_response = cache_output_tools.retrieve_output("extract_and_structure_data", prompt_hash)
+    if cached_response is not None:
+        return cached_response
+    
+    # Run the assistant and cache the output if successful
+    output = await run_assistant(client, assistant, thread, formatted_prompt, response_type, allowed_tools)
+    if output and output != 'FAIL':
+        cache_output_tools.cache_output("extract_and_structure_data", prompt_hash, output)
+    
+    return output
+
+class RowItem(BaseModel):
+    column_value: str
+    
+class GenericList(BaseModel):
+    rows: List[RowItem]
+    
+def lookup_response_type(name: str):
+    for model in GLOBAL_DATA_MODELS:
+        if model.__name__ == name:
+            return model
+    return GenericList  # Default response type
+
+
+async def process_agent_request(row_batch: List[Dict], workflow: Dict, custom_instructions: str) -> List[Dict]:
+    """
+    Process agent request using the OpenAI client.
+    """
+    #TODO: handle timezone here.
+    logging.getLogger("openai").setLevel(logging.WARNING)
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+
+    todays_date = datetime.now(timezone.utc).isoformat()        
+    todays_day = datetime.now(timezone.utc).strftime('%d')    
+    instructions = f"Hi, You are an AI Assistant. Help the user with their tasks.\n\n Todays date is: {todays_date} Todays day is {todays_day} \n\n{custom_instructions}\n\n"
+    try:
+        client = AsyncOpenAI()
+        assistant = await client.beta.assistants.create(
+            name="AI Assistant",
+            instructions=instructions,   
+            tools=[],
+            model="gpt-5.1-chat"
+        )
+        thread = await client.beta.threads.create()
+        parsed_outputs = []
+        task_outputs = {}  # Dictionary to store outputs of tasks
+        input_list = {}
+        input_list['initial_input_list'] = {
+            "data": row_batch,
+            "format": "list"
+            }
+        task_outputs['initial_input'] = input_list
+        for task in workflow['tasks']:
+            # Process each task
+            task_outputs = await process_task(client, assistant, thread, task, task_outputs)
+        # Collect the final output
+        parsed_outputs.append(task_outputs)
+        return parsed_outputs
+    except Exception as e:
+        logging.warning(f"process_agent_request An error occurred: {e}", exc_info=True)
+        return [{"error": f"Error Processing Leads. process_agent_request process_agent_request An error occurred: {e}"}]
+    finally:
+        try:
+            await client.beta.assistants.delete(assistant.id)
+        except Exception as e:
+            logging.info(f"Error deleting assistant: {e}")
+
+
+async def process_task(client, assistant, thread, task, task_outputs):
+    """
+    Process a single task in the workflow.
+    """
+    # Prepare inputs
+    task_inputs = await prepare_task_inputs(task, task_outputs)
+
+    # Run the operation
+    output = await run_task_operation(client, assistant, thread, task, task_inputs)
+
+    # Store outputs
+    await store_task_outputs(task, output, task_outputs)
+
+    return task_outputs
+
+async def read_csv_rows(file_path):
+    rows = []
+    with open(file_path, mode='r') as file:
+        csv_reader = csv.reader(file)
+        for row in csv_reader:
+            rows.append(row)
+    return rows
+
+async def prepare_task_inputs(task, task_outputs):
+    """
+    Prepare the inputs for a task based on its input specifications.
+    """
+    inputs = task.get('inputs', {})
+    task_inputs = {}
+    for input_name, input_spec in inputs.items():
+        source = input_spec.get('source', {})
+        source_type = source.get('type', '')
+        format = input_spec.get('format', 'list')
+        if source_type == 'inline':
+            # Get from inline source
+            input_data = source.get('data')
+        elif source_type == 'task_output':
+            # Get from previous task output
+            task_id = source.get('task_id')
+            output_key = source.get('output_key')
+            previous_task_output = task_outputs.get(task_id, {})            
+            if isinstance(previous_task_output, dict):
+                output_item = previous_task_output.get(output_key)
+                input_data = output_item['data']
+            else:
+                input_data = previous_task_output
+        
+            # Ensure input_data is a list
+            if not isinstance(input_data, list):
+                input_data = [input_data]
+        elif source_type == 'google_drive':
+            # Handle Google Drive source
+            path = source.get('location')
+            input_data_path = await read_from_google_drive(path)
+            input_data = await read_csv_rows(input_data_path)
+        elif source_type == 'local_path':
+            # Handle local path source
+            input_data_path = source.get('location')
+            input_data = await read_csv_rows(input_data_path)
+        else:
+            input_data = None
+        if input_data:
+            task_inputs[input_name] = { 
+                                        "format": format, 
+                                        "data" : input_data
+            }
+    return task_inputs
+
+async def process_using_ai_assistant(prompt_template, task_inputs, client, assistant, thread, response_type, allowed_tools, task):
+    outputs = []
+    for key, value in task_inputs.items():
+        format = value.get('format', 'list')
+        items = value.get('data')
+        if format == 'list':
+            for item in items:
+                formatted_prompt = prompt_template.replace(
+                    "{{ inputs." + key + " }}", json.dumps(item)
+                )
+                # Run assistant with prompt
+                logging.info(formatted_prompt)
+                output = await extract_and_structure_data(
+                    client, assistant, thread, formatted_prompt, task_inputs, response_type, allowed_tools
+                )
+                if output and output == 'FAIL':
+                    pass
+                output_json = None
+                if isinstance(output, str):
+                    try:
+                        output_json = json.loads(output)
+                    except json.JSONDecodeError:
+                        pass
+                if (
+                    output_json
+                    and isinstance(output_json, dict)
+                    and 'data' in output_json
+                    and isinstance(output_json['data'], list)
+                ):
+                    # Deserialize the JSON to responseType
+                    items_deserialized = [response_type.parse_obj(item) for item in output_json['data']]
+                    # Iterate over items_deserialized
+                    for item in items_deserialized:
+                        # Serialize each item back to JSON
+                        serialized_item = json.dumps(item.dict())
+                        outputs.append(serialized_item)
+                elif output_json and isinstance(output_json, dict):
+                    output_deserialized = response_type.parse_obj(output_json)
+                    outputs.append(json.dumps(output_deserialized.dict()))
+                else:
+                    logging.warning("output_json is None or not a dict")
+                if outputs and len(outputs) > 0:
+                    interim_return_val = {
+                        "data": outputs,
+                        "format": "list"
+                    }
+                    await store_task_outputs_interim_checkpoint(task, interim_return_val, task_inputs)
+        else:
+            # Handle other formats if necessary
+            pass
+    return outputs
+
+async def process_transform_json(task_inputs, response_type, task):
+    outputs = []
+    task_id = task.get('id')
+    for input_name, input_info in task_inputs.items():
+        data_format = input_info.get('format', 'list')
+        input_info.get('transform_function_name', f"{task_id}_transform_input_json")
+        items = input_info.get('data')
+        if data_format == 'list':
+            if items and len(items) > 0:
+                # Generate the transformation function
+                # if GLOBAL_GENERATED_PYTHON_CODE.get(transform_function_name, ''):
+                #     transformation_function = GLOBAL_GENERATED_PYTHON_CODE[transform_function_name]
+                # else:
+                #     function_name = await transform_json_with_type(
+                #         items[0],
+                #         response_type,
+                #         transform_function_name
+                #     )
+                #     transformation_function = GLOBAL_GENERATED_PYTHON_CODE[function_name]
+                transformation_function = lambda x: x
+                for item in items:
+                    input_json = json.loads(item)
+                    output_json = transformation_function(input_json)
+                    output_deserialized = response_type.parse_obj(output_json)
+                    outputs.append(json.dumps(output_deserialized.dict()))
+                if outputs:
+                    interim_return_val = {
+                        "data": outputs,
+                        "format": "list"
+                    }
+                    await store_task_outputs_interim_checkpoint(task, interim_return_val, task_inputs)
+        else:
+            # Handle other formats if necessary
+            pass
+    return outputs
+
+async def process_function_call(operation, task_inputs, outputs):
+    function_name = operation.get('function', '')
+    args = operation.get('args', [])
+    function = GLOBAL_TOOLS_FUNCTIONS.get(function_name)
+    if function is None:
+        raise Exception(f"Function {function_name} not found.")
+
+    for key, value in task_inputs.items():
+        format = value.get('format', 'list')
+        items = value.get('data')
+        item_parse_args_with_llm = operation.get('args_llm_parsed', 'False')
+        if format == 'list':
+            for item in items:
+                # Prepare function keyword arguments
+                if item_parse_args_with_llm == 'True':
+                    function_kwargs, status = await get_function_call_arguments(
+                        item, function_name
+                    )
+                    if status == 'FAIL':
+                        continue
+                else:
+                    function_kwargs = {arg: item.get(arg, '') for arg in args}
+                if asyncio.iscoroutinefunction(function):
+                    output = await function(**function_kwargs)
+                else:
+                    output = function(**function_kwargs)
+                process_output(output, outputs)
+        else:
+            # Prepare function arguments
+            function_kwargs = {
+                arg: task_inputs.get(arg, {}).get("data", '') for arg in args
+            }
+            if asyncio.iscoroutinefunction(function):
+                output = await function(**function_kwargs)
+            else:
+                output = function(**function_kwargs)
+            process_output(output, outputs)
+    return outputs
+
+def process_output(output, outputs):
+    if isinstance(output, list):
+        for item in output:
+            if isinstance(item, dict):
+                outputs.append(json.dumps(item))
+            else:
+                outputs.append(item)
+    else:
+        outputs.append(output)
+                 
+async def run_task_operation(client, assistant, thread, task, task_inputs):
+    """
+    Execute the operation defined in the task.
+    """
+    operation = task.get('operation', {})
+    operation_type = operation.get('type', '')
+    allowed_tools = operation.get('allowed_tools', [])
+    response_type_name = operation.get('response_type', 'GenericList')
+    response_type = lookup_response_type(response_type_name)
+    outputs = []
+
+    if operation_type == 'ai_assistant_call':
+        prompt_template = operation.get('prompt', '')
+        outputs = await process_using_ai_assistant(
+            prompt_template, task_inputs, client, assistant, thread, response_type, allowed_tools, task
+        )
+    elif operation_type == 'ai_transform_input_json':
+        outputs = await process_transform_json(
+            task_inputs, response_type, task
+        )
+    elif operation_type == 'python_callable':
+        outputs = await process_function_call(operation, task_inputs, outputs)
+    return {
+        "data": outputs,
+        "format": "list"
+    }
+
+async def store_task_outputs_interim_checkpoint(task, output, task_outputs):
+    """
+    Store the outputs of a task for use in subsequent tasks.
+    """
+    outputs = task.get('outputs', {})
+    if outputs:
+        for output_name, output_spec in outputs.items():
+            destination = output_spec.get('destination', {})
+            if destination:
+                dest_type = destination.get('type')
+                path_template = destination.get('path_template')
+                if path_template:
+                    current_timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
+                    current_timestamp = '_interim'
+                    path = path_template.replace('{timestamp}', current_timestamp)
+                    path = path.replace('{task_id}', task['id'])
+                    local_path = path
+
+                    if dest_type == 'google_drive':
+                        local_path = os.path.join('/tmp', task['id'], path)
+
+                    if dest_type == 'google_drive' or dest_type == 'local_path':
+                        directory = os.path.dirname(local_path)
+                        if directory and not os.path.exists(directory):
+                            os.makedirs(directory)
+                        logging.info(f"Writing output to {local_path}\n")
+
+                        if output.get("format", "") == 'list':
+                            data_list = []
+                            for item in output.get("data", []):
+                                try:
+                                    data_list.append(json.loads(item))
+                                except json.JSONDecodeError:
+                                    # Handle or log the invalid JSON item
+                                    pass
+                            # Write the full list first with a 'full_list' prefix
+                            def get_prefixed_path(file_path, prefix):
+                                directory, filename = os.path.split(file_path)
+                                name, ext = os.path.splitext(filename)
+                                prefixed_filename = f"{prefix}_{name}{ext}"
+                                return os.path.join(directory, prefixed_filename)
+
+                            full_list_local_path = get_prefixed_path(local_path, 'full_list')
+                            full_list_directory = os.path.dirname(full_list_local_path)
+                            if full_list_directory and not os.path.exists(full_list_directory):
+                                os.makedirs(full_list_directory)
+                            logging.info(f"Writing full list output to {full_list_local_path}\n")
+
+                            with open(full_list_local_path, 'w') as full_file:
+                                if data_list and len(data_list) > 0:
+                                    headers = [key for key in data_list[0].keys()]
+                                    writer = csv.DictWriter(full_file, fieldnames=headers)
+                                    writer.writeheader()
+                                    for data in data_list:
+                                        filtered_data = {key: value for key, value in data.items() if key in headers}
+                                        writer.writerow(filtered_data)
+                                else:
+                                    writer = csv.DictWriter(full_file, fieldnames=[])
+                                    writer.writeheader()
+    return task_outputs
+
+def filter_data_list(data_list, filter_by):
+    """
+    Filter the data_list based on conditions specified in filter_by.
+    Supported operators: 'gt', 'lt', 'eq', 'gte', 'lte', 'ne'
+    """
+    from operator import gt, lt, eq, ge, le, ne
+
+    operator_map = {
+        'gt': gt,
+        'lt': lt,
+        'eq': eq,
+        'gte': ge,
+        'lte': le,
+        'ne': ne
+    }
+
+    filtered_list = []
+    for item in data_list:
+        include_item = True
+        for property_name, conditions in filter_by.items():
+            value = item.get(property_name)
+            if value is None or (isinstance(value, str) and value.strip() == ""):
+                include_item = False
+                break
+            for op, compare_value in conditions.items():
+                op_func = operator_map.get(op)
+                if op_func is None:
+                    continue  # Unsupported operator
+                try:
+                    # Convert values to float for comparison if possible
+                    item_value = float(value)
+                    compare_value = float(compare_value)
+                except (ValueError, TypeError):
+                    item_value = value
+                if not op_func(item_value, compare_value):
+                    include_item = False
+                    break
+            if not include_item:
+                break
+        if include_item:
+            filtered_list.append(item)
+    return filtered_list
+
+def convert_value(value):
+    """
+    Convert the value to the appropriate type for sorting.
+    """
+    if value is None or (isinstance(value, str) and value.strip() == ""):
+        return ""
+    try:
+        return float(value)
+    except ValueError:
+        pass
+    try:
+        return datetime.fromisoformat(value)
+    except ValueError:
+        pass
+    return str(value)
+
+def filter_and_sort(data_list, output_spec):
+    """
+    Filter and sort the data_list based on the output_spec.
+    """
+    required_properties = output_spec.get('required_properties', [])
+    if required_properties:
+        data_list = remove_empty_property_rows(data_list, required_properties)
+    
+    dedup_by = output_spec.get('deduplication_properties', [])
+    if dedup_by:
+        data_list = deduplicate_list_by_properties(data_list, dedup_by)
+    
+    sort_by_asc = output_spec.get('sort_by_asc', [])
+    sort_by_desc = output_spec.get('sort_by_desc', [])
+    
+    # Combine sort fields with their corresponding order
+    sort_fields = [(key, True) for key in sort_by_asc] + [(key, False) for key in sort_by_desc]
+    logging.info(f"Sorting by: {sort_fields}")
+    
+    # Sort from least significant to most significant key
+    for key, ascending in reversed(sort_fields):
+        data_list = sorted(
+            data_list,
+            key=lambda x: convert_value(x.get(key)),
+            reverse=not ascending
+        )
+    
+    filter_by = output_spec.get('filter_by', {})
+    if filter_by:
+        data_list = filter_data_list(data_list, filter_by)
+    
+    return data_list
+
+# Store the output of a task run.
+async def store_task_outputs(task, output, task_outputs):
+    """
+    Store the outputs of a task for use in subsequent tasks.
+    """
+    outputs = task.get('outputs', {})
+    if outputs:
+        for output_name, output_spec in outputs.items():
+            # Store output in task_outputs using task id and output_name
+            if task['id'] not in task_outputs:
+                task_outputs[task['id']] = {}
+
+            destination = output_spec.get('destination', {})
+            if destination:
+                dest_type = destination.get('type')
+                path_template = destination.get('path_template')
+                if path_template:
+                    current_timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
+                    path = path_template.replace('{timestamp}', current_timestamp)
+                    path = path.replace('{task_id}', task['id'])
+                    local_path = path
+
+                    if dest_type == 'google_drive':
+                        local_path = os.path.join('/tmp', task['id'], path)
+
+                    if dest_type == 'google_drive' or dest_type == 'local_path':
+                        directory = os.path.dirname(local_path)
+                        if directory and not os.path.exists(directory):
+                            os.makedirs(directory)
+                        logging.info(f"Writing output to {local_path}\n")
+
+                        if output.get("format", "") == 'list':
+                            data_list = []
+                            for item in output.get("data", []):
+                                try:
+                                    data_list.append(json.loads(item))
+                                except json.JSONDecodeError:
+                                    # Handle or log the invalid JSON item
+                                    pass
+                            logging.info(f"Total count: {len(data_list)}")
+
+                            # Write the full list first with a 'full_list' prefix
+                            def get_prefixed_path(file_path, prefix):
+                                directory, filename = os.path.split(file_path)
+                                name, ext = os.path.splitext(filename)
+                                prefixed_filename = f"{prefix}_{name}{ext}"
+                                return os.path.join(directory, prefixed_filename)
+
+                            full_list_local_path = get_prefixed_path(local_path, 'full_list')
+                            full_list_directory = os.path.dirname(full_list_local_path)
+                            if full_list_directory and not os.path.exists(full_list_directory):
+                                os.makedirs(full_list_directory)
+                            logging.info(f"Writing full list output to {full_list_local_path}\n")
+
+                            with open(full_list_local_path, 'w') as full_file:
+                                if data_list and len(data_list) > 0:
+                                    headers = [key for key in data_list[0].keys()]
+                                    writer = csv.DictWriter(full_file, fieldnames=headers)
+                                    writer.writeheader()
+                                    for data in data_list:
+                                        filtered_data = {key: value for key, value in data.items() if key in headers}
+                                        writer.writerow(filtered_data)
+                                else:
+                                    writer = csv.DictWriter(full_file, fieldnames=[])
+                                    writer.writeheader()
+
+                            if data_list and len(data_list) > 0:
+                                data_list = filter_and_sort(data_list, output_spec)
+                                if data_list and len(data_list) > 0:
+                                    logging.info(f"Deduped and removed count: {len(data_list)}")
+                                    headers = [key for key, value in data_list[0].items() if isinstance(value, (str, int, float, bool))]
+                                    with open(local_path, 'w') as file:
+                                        writer = csv.DictWriter(file, fieldnames=headers)
+                                        writer.writeheader()
+                                        for data in data_list:
+                                            filtered_data = {key: value for key, value in data.items() if key in headers}
+                                            writer.writerow(filtered_data)
+                                else:
+                                    writer = csv.DictWriter(full_file, fieldnames=[])
+                                    writer.writeheader()
+                            else:
+                                with open(local_path, 'w') as file:
+                                    writer = csv.DictWriter(file, fieldnames=[])
+                                    writer.writeheader()
+                        else:
+                            with open(local_path, 'w') as file:
+                                file.write(str(output))
+                    else:
+                        pass
+                if dest_type == 'google_drive':
+                    await write_to_google_drive(path, local_path)
+
+                task_outputs[task['id']][output_name] = output
+    else:
+        task_outputs[task['id']] = output
+
+# Remove rows with None or empty values for the specified properties.
+def remove_empty_property_rows(data_list, properties):
+    """
+    Remove rows with None or empty values for the specified properties.
+    """
+    filtered_list = []
+    for item in data_list:
+        empty = False
+        for property_name in properties:
+            value = item.get(property_name)
+            if value is None or (isinstance(value, str) and value.strip() == ""):
+                empty = True
+                break
+        if not empty:
+            filtered_list.append(item)
+    return filtered_list
+
+# Deduplicate list by given input properties.
+def deduplicate_list_by_properties(data_list, properties):
+    """
+    Deduplicate a list of dictionaries by a list of properties in order.
+    Only deduplicate if the property value is not None or empty, strip spaces, and compare in lowercase.
+    """
+    for property_name in properties:
+        seen = set()
+        deduplicated_list = []
+        for item in data_list:
+            value = item.get(property_name)
+            value = str(value or "").strip().lower()
+            if value == "":
+                deduplicated_list.append(item)
+            elif value not in seen:
+                seen.add(value)
+                deduplicated_list.append(item)
+        data_list = deduplicated_list
+    return data_list
+
+async def write_to_google_drive(cloud_path, local_path):
+    # Placeholder function for writing to Google Drive
+    await write_content_to_googledrive(cloud_path, local_path)
+    logging.info(f"Writing to Google Drive at {cloud_path} {local_path}")
+
+
+# Get a dynamic pyndantic model that corresponds to the function signature
+def get_dynamic_model(function_name: str, function: Callable) -> Type[BaseModel]:
+    """
+    Dynamically creates a Pydantic BaseModel subclass based on the parameters of the given function.
+
+    Args:
+        function_name (str): The name of the function.
+        function (Callable): The function object.
+
+    Returns:
+        Type[BaseModel]: A dynamically created Pydantic model class.
+    """
+    # Retrieve the function's signature
+    signature = inspect.signature(function)
+    fields = {}
+
+    for param_name, param in signature.parameters.items():
+        # Extract the parameter's type annotation
+        annotation = param.annotation if param.annotation is not inspect.Parameter.empty else Any
+
+        # Determine if the parameter has a default value
+        if param.default is not inspect.Parameter.empty:
+            default_value = param.default
+        else:
+            default_value = ...
+
+        # Create a Field with a description
+        field_info = Field(
+            default=default_value,
+            description=f"Parameter '{param_name}' of type '{annotation.__name__}'"
+        )
+
+        # Add the field to the fields dictionary
+        fields[param_name] = (annotation, field_info)
+
+    # Create and return the dynamic model
+    dynamic_model = create_model(
+        f"{function_name}_Arguments",
+        __base__=BaseModel,
+        **fields
+    )
+
+    return dynamic_model
+
+# Given a function definition and input string extract the function call arguments using OpenAI API
+async def get_function_call_arguments(input_text: str, function_name: str) -> Tuple[Dict[str, Any], str]:
+    """
+    Extracts function call arguments from the input text using OpenAI's API.
+
+    Args:
+        input_text (str): The input text containing the function call.
+        function_name (str): The name of the function to extract arguments for.
+
+    Returns:
+        Tuple[Dict[str, Any], str]: A tuple containing the function arguments as a dictionary and a status message.
+    """
+    try:
+        # Retrieve the function and its parameters
+        function, _ = get_function(function_name)
+        
+        # Generate a dynamic Pydantic model based on the function's parameters
+        dynamic_model = get_dynamic_model(function_name, function)
+        
+        # Define the tool using the dynamic model
+        tool = pydantic_function_tool(dynamic_model)
+        
+        # Construct the prompt
+        prompt = f"Extract the arguments for the function '{function_name}' from the following input:\n\n{input_text}"
+        
+        # Initialize the OpenAI client
+        client = AsyncOpenAI()
+        
+        # Make the API call
+        response = await client.beta.chat.completions.parse(
+            model="gpt-5.1-chat",
+            messages=[
+                {"role": "system", "content": "Extract function arguments in JSON format."},
+                {"role": "user", "content": prompt},
+            ],
+            tools=[tool],
+            response_format=dynamic_model
+        )
+        
+        # Extract the function call arguments from the response
+        parsed_output = vars(response.choices[0].message.parsed)
+        return parsed_output, 'SUCCESS'
+    
+    except OpenAIError as e:
+        logging.error(f"OpenAI API error: {e}")
+        raise HTTPException(status_code=502, detail="Error communicating with the OpenAI API.")
+    except Exception as e:
+        logging.error(f"Unexpected error: {e}")
+        raise HTTPException(status_code=500, detail="An unexpected error occurred while processing your request.")
+