lionagi 0.0.316__py3-none-any.whl → 0.1.1__py3-none-any.whl

lionagi/experimental/directive/evaluator/base_evaluator.py

@@ -0,0 +1,202 @@
+import ast
+import operator
+from typing import Any, Dict, Tuple, Callable
+
+from lionagi.libs.ln_convert import to_dict
+
+
+class BaseEvaluator:
+    """
+    A class to evaluate mathematical and boolean expressions from strings using Python's AST.
+
+    Attributes:
+        allowed_operators (Dict[type, Any]): A dictionary mapping AST node types to their corresponding Python operator functions.
+        cache (Dict[Tuple[str, Tuple], Any]): A dictionary used to cache the results of evaluated expressions and sub-expressions.
+    """
+
+    def __init__(self) -> None:
+        """Initializes the evaluator with supported operators and an empty cache."""
+        self.allowed_operators: Dict[type, Any] = {
+            ast.Add: operator.add,
+            ast.Sub: operator.sub,
+            ast.Mult: operator.mul,
+            ast.Div: operator.truediv,
+            ast.Pow: operator.pow,
+            ast.Mod: operator.mod,
+            ast.Eq: operator.eq,
+            ast.NotEq: operator.ne,
+            ast.Lt: operator.lt,
+            ast.LtE: operator.le,
+            ast.Gt: operator.gt,
+            ast.GtE: operator.ge,
+            ast.And: lambda x, y: x and y,
+            ast.Or: lambda x, y: x or y,
+            ast.Not: operator.not_,
+            ast.USub: operator.neg,
+        }
+        self.cache: Dict[Tuple[str, Tuple], Any] = {}
+
+    def evaluate(self, expression: str, context: Dict[str, Any]) -> Any:
+        """
+        Evaluates a given expression string using the provided context.
+
+        Args:
+            expression (str): The expression to evaluate.
+            context (Dict[str, Any]): A dictionary mapping variable names to their values.
+
+        Returns:
+            Any: The result of the evaluated expression.
+
+        Raises:
+            ValueError: If the expression cannot be evaluated.
+        """
+        cache_key = (expression, tuple(sorted(context.items())))
+        if cache_key in self.cache:
+            return self.cache[cache_key]
+
+        try:
+            tree = ast.parse(expression, mode="eval")
+            result = self._evaluate_node(tree.body, context)
+            self.cache[cache_key] = result
+            return result
+        except Exception as e:
+            raise ValueError(f"Failed to evaluate expression: {expression}. Error: {e}")
+
+    def _evaluate_node(self, node: ast.AST, context: Dict[str, Any]) -> Any:
+        """Recursively evaluates an AST node."""
+        if isinstance(node, ast.BinOp):
+            left = self._evaluate_node(node.left, context)
+            op_func = self.allowed_operators[type(node.op)]
+            right = self._evaluate_node(node.right, context)
+            result = op_func(left, right)
+        elif isinstance(node, ast.UnaryOp):
+            operand = self._evaluate_node(node.operand, context)
+            result = self.allowed_operators[type(node.op)](operand)
+        elif isinstance(node, ast.Name):
+            result = context.get(node.id, None)
+        elif isinstance(node, ast.Constant):
+            result = node.value
+        elif isinstance(node, ast.Compare):
+            left = self._evaluate_node(node.left, context)
+            result = True
+            for operation, comparator in zip(node.ops, node.comparators):
+                op_func = self.allowed_operators[type(operation)]
+                right = self._evaluate_node(comparator, context)
+                result = result and op_func(left, right)
+                if not result:
+                    break
+                left = right
+        elif isinstance(node, ast.BoolOp):
+            values = [self._evaluate_node(value, context) for value in node.values]
+            if isinstance(node.op, ast.And):
+                result = all(values)
+            elif isinstance(node.op, ast.Or):
+                result = any(values)
+            else:
+                raise ValueError("Unsupported boolean operation.")
+        else:
+            raise ValueError("Unsupported operation in condition.")
+        return result
+
+    def add_custom_operator(self, operator_name, operation_func):
+        """Adds a custom operator to the evaluator."""
+        custom_node_class = type(operator_name, (ast.AST,), {})
+        if custom_node_class not in self.allowed_operators:
+            self.allowed_operators[custom_node_class] = operation_func
+        else:
+            raise ValueError(f"Custom operator '{operator_name}' is already defined.")
+
+    def evaluate_file(self, file_path, context, format="line"):
+        """Evaluates expressions from a file."""
+        if format == "line":
+            with open(file_path, "r") as file:
+                last_result = None
+                for line in file:
+                    line = line.strip()
+                    if line:
+                        last_result = self.evaluate(line, context)
+                return last_result
+        elif format == "json":
+            with open(file_path, "r") as file:
+                data = to_dict(file)
+                last_result = None
+                for expression in data:
+                    last_result = self.evaluate(expression, context)
+                return last_result
+        else:
+            raise ValueError(f"Unsupported file format: {format}")
+
+    def validate_expression(self, expression):
+        """Validates the given expression."""
+        try:
+            tree = ast.parse(expression, mode="eval")
+            self._validate_node(tree.body)
+            return True, "Expression is valid."
+        except Exception as e:
+            return False, f"Invalid expression: {str(e)}"
+
+    def _validate_node(self, node):
+        """Validates an AST node."""
+        if isinstance(
+            node, (ast.BinOp, ast.Compare, ast.BoolOp, ast.Name, ast.Constant)
+        ):
+            if (
+                isinstance(node, ast.BinOp)
+                and type(node.op) not in self.allowed_operators
+            ):
+                raise ValueError(
+                    f"Operation {type(node.op).__name__} is not supported."
+                )
+        else:
+            raise ValueError("Unsupported node type in expression.")
+
+
+class BaseEvaluationEngine:
+    def __init__(self) -> None:
+        self.variables: Dict[str, Any] = {}
+        self.functions: Dict[str, Callable] = {
+            "print": print,
+        }
+
+    def _evaluate_expression(self, expression: str) -> Any:
+        try:
+            return eval(expression, {}, self.variables)
+        except NameError as e:
+            raise ValueError(f"Undefined variable. {e}")
+
+    def _assign_variable(self, var_name: str, value: Any) -> None:
+        self.variables[var_name] = value
+
+    def _execute_function(self, func_name: str, *args: Any) -> None:
+        if func_name in self.functions:
+            self.functions[func_name](*args)
+        else:
+            raise ValueError(f"Function {func_name} not defined.")
+
+    def _execute_statement(self, stmt: ast.AST) -> None:
+        if isinstance(stmt, ast.Assign):
+            var_name = stmt.targets[0].id
+            value = self._evaluate_expression(ast.unparse(stmt.value))
+            self._assign_variable(var_name, value)
+        elif isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call):
+            func_name = stmt.value.func.id
+            args = [
+                self._evaluate_expression(ast.unparse(arg)) for arg in stmt.value.args
+            ]
+            self._execute_function(func_name, *args)
+        elif isinstance(stmt, ast.For):
+            iter_var = stmt.target.id
+            if isinstance(stmt.iter, ast.Call) and stmt.iter.func.id == "range":
+                start, end = [
+                    self._evaluate_expression(ast.unparse(arg))
+                    for arg in stmt.iter.args
+                ]
+                for i in range(start, end):
+                    self.variables[iter_var] = i
+                    for body_stmt in stmt.body:
+                        self._execute_statement(body_stmt)
+
+    def execute(self, script: str) -> None:
+        tree = ast.parse(script)
+        for stmt in tree.body:
+            self._execute_statement(stmt)

lionagi/experimental/directive/evaluator/sandbox_.py

@@ -0,0 +1,14 @@
+# filename: enhanced_script_engine.py
+import ast
+
+
+class SandboxTransformer(ast.NodeTransformer):
+    """AST transformer to enforce restrictions in sandbox mode."""
+
+    def visit_Import(self, node):
+        raise RuntimeError("Import statements are not allowed in sandbox mode.")
+
+    def visit_Exec(self, node):
+        raise RuntimeError("Exec statements are not allowed in sandbox mode.")
+
+    # Add other visit methods for disallowed operations or nodes

lionagi/experimental/directive/evaluator/script_engine.py

@@ -0,0 +1,83 @@
+import ast
+from functools import lru_cache
+from lionagi.libs import AsyncUtil
+from .base_evaluator import BaseEvaluator
+from .sandbox_ import SandboxTransformer
+
+
+class ScriptEngine:
+    def __init__(self):
+        self.variables = {}
+        self.safe_evaluator = BaseEvaluator()
+        self.functions = {
+            "processData": self.process_data,
+        }
+
+    def process_data(self, data):
+        return data * 2
+
+    def _evaluate_expression(self, expression):
+        return self.safe_evaluator.evaluate(expression, self.variables)
+
+    def _assign_variable(self, var_name, value):
+        self.variables[var_name] = value
+
+    def _execute_function(self, func_name, arg):
+        if func_name in self.functions:
+            return self.functions[func_name](arg)
+        else:
+            raise ValueError(f"Function {func_name} not defined.")
+
+    def execute(self, script):
+        tree = ast.parse(script)
+        for stmt in tree.body:
+            if isinstance(stmt, ast.Assign):
+                var_name = stmt.targets[0].id
+                value = self._evaluate_expression(ast.unparse(stmt.value))
+                self._assign_variable(var_name, value)
+            elif isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call):
+                func_name = stmt.value.func.id
+                arg = self._evaluate_expression(ast.unparse(stmt.value.args[0]))
+                result = self._execute_function(func_name, arg)
+                # For demonstration, manually update 'x' to simulate expected behavior
+                if func_name == "processData":
+                    self._assign_variable("x", result)
+
+    def add_hook(self, event_name, callback):
+        """Subscribe a callback function to a specific event."""
+        self.hooks[event_name].append(callback)
+
+    def trigger_hooks(self, event_name, *args, **kwargs):
+        """Trigger all callbacks attached to a specific event."""
+        for callback in self.hooks[event_name]:
+            callback(*args, **kwargs)
+
+    async def process_data(self, data):
+        # Example asynchronous function
+        return data * 2
+
+    @lru_cache(maxsize=128)
+    def evaluate_expression(self, expression):
+        return self.safe_evaluator.evaluate(expression, self.variables)
+
+    async def _execute_function_async(self, func_name, arg):
+        if func_name in self.functions:
+            func = self.functions[func_name]
+            if AsyncUtil.is_coroutine_func(func):
+                return await func(arg)
+            else:
+                return func(arg)
+        else:
+            raise ValueError(f"Function {func_name} not defined.")
+
+    def execute_sandboxed(self, script):
+        # Parse and sanitize the script
+        tree = ast.parse(script, mode="exec")
+        sanitized_tree = SandboxTransformer().visit(tree)
+        ast.fix_missing_locations(sanitized_tree)
+
+        # Compile the sanitized AST
+        code = compile(sanitized_tree, "<sandbox>", "exec")
+
+        # Execute the code in a restricted namespace
+        exec(code, {"__builtins__": None}, self.variables)

lionagi/experimental/directive/parser/base_parser.py

@@ -0,0 +1,215 @@
+from typing import List, Optional
+
+from lionagi.libs.ln_tokenizer import BaseToken
+from ..schema import IfNode, TryNode, ForNode
+
+
+class BaseDirectiveParser:
+    """A base parser with lookahead, error recovery, and backtracking support.
+
+    Attributes:
+        tokens (List[BaseToken]): A list of tokens to be parsed.
+        current_token_index (int): The index of the current token in the tokens list.
+        current_token (Optional[BaseToken]): The current token being processed.
+
+    Examples:
+        >>> tokenizer = BaseTokenizer("IF x > 10 THEN DO something ENDIF")
+        >>> tokens = tokenizer.get_tokens()
+        >>> parser = BaseParser(tokens)
+        >>> print(parser.current_token)
+        BaseToken(KEYWORD, IF)
+    """
+
+    def __init__(self, tokens: List[BaseToken]):
+        self.tokens = tokens
+        self.current_token_index = -1
+        self.current_token: Optional[BaseToken] = None
+        self.next_token()
+
+    def next_token(self) -> None:
+        """Advances to the next token in the list."""
+        self.current_token_index += 1
+        if self.current_token_index < len(self.tokens):
+            self.current_token = self.tokens[self.current_token_index]
+        else:
+            self.current_token = None
+
+    def peek_next_token(self, offset: int = 1) -> BaseToken | None:
+        """Peeks at the next token without consuming it.
+
+        Args:
+            offset (int): The number of tokens to look ahead.
+
+        Returns:
+            Optional[BaseToken]: The token at the specified lookahead offset, or None if end of list.
+        """
+        peek_index = self.current_token_index + offset
+        if peek_index < len(self.tokens):
+            return self.tokens[peek_index]
+        else:
+            return None
+
+    def skip_until(self, token_types: List[str]) -> None:
+        """Skips tokens until a token of the specified type is found.
+
+        Args:
+            token_types (List[str]): A list of token types to stop skipping.
+        """
+        while self.current_token and self.current_token.type not in token_types:
+            self.next_token()
+
+    def mark(self) -> int:
+        """Marks the current position in the token list for potential backtracking.
+
+        Returns:
+            int: The current token index.
+        """
+        return self.current_token_index
+
+    def reset_to_mark(self, mark: int) -> None:
+        """Resets the parser to a previously marked position.
+
+        Args:
+            mark (int): The token index to reset to.
+        """
+        self.current_token_index = mark - 1
+        self.next_token()
+
+    def skip_semicolon(self):
+        if self.current_token and self.current_token.value == ";":
+            self.next_token()
+
+    def parse_expression(self):
+        expr = ""
+        while self.current_token and self.current_token.value != ";":
+            expr += self.current_token.value + " "
+            self.next_token()
+        # Expecting a semicolon at the end of the condition
+        if self.current_token.value != ";":
+            raise SyntaxError("Expected ';' at the end of the condition")
+        self.next_token()  # Move past the semicolon to the next part of the statement
+        return expr.strip()
+
+    def parse_if_block(self):
+        block = []
+        # Parse the block until 'ELSE', 'ENDIF', ensuring not to include semicolons as part of the block
+        while self.current_token and self.current_token.value not in ("ENDIF", "ELSE"):
+            if self.current_token.value == "DO":
+                self.next_token()  # Move past 'DO' to get to the action
+            block.append(self.current_token.value)  # Add the action to the block
+            self.next_token()  # Move to the next token, which could be a semicolon or the next action
+            if self.current_token.value == ";":
+                self.next_token()  # Move past the semicolon
+        return block
+
+    def parse_if_statement(self):
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "IF":
+            raise SyntaxError("Expected IF statement")
+        self.next_token()  # Skip 'IF'
+
+        condition = self.parse_expression()  # Now properly ends after the semicolon
+
+        true_block = []
+        if self.current_token.value == "DO":
+            true_block = self.parse_if_block()  # Parse true block after 'DO'
+
+        false_block = None
+        if self.current_token and self.current_token.value == "ELSE":
+            self.next_token()  # Skip 'ELSE', expect 'DO' next for the false block
+            self.skip_semicolon()
+            if self.current_token.value != "DO":
+                raise SyntaxError("Expected 'DO' after 'ELSE'")
+            self.next_token()  # Skip 'DO'
+            false_block = self.parse_if_block()  # Parse false block
+
+        return IfNode(condition, true_block, false_block)
+
+    def parse_for_statement(self):
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "FOR":
+            raise SyntaxError("Expected FOR statement")
+        self.next_token()  # Skip 'FOR'
+
+        # Parse the iterator variable
+        if self.current_token.type != "IDENTIFIER":
+            raise SyntaxError("Expected iterator variable after FOR")
+        iterator = self.current_token.value
+        self.next_token()  # Move past the iterator variable
+
+        # Expect and skip 'IN' keyword
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "IN":
+            raise SyntaxError("Expected 'IN' after iterator variable")
+        self.next_token()  # Move past 'IN'
+
+        # Parse the collection
+        if self.current_token.type not in ["IDENTIFIER", "LITERAL"]:
+            raise SyntaxError("Expected collection after 'IN'")
+        collection = self.current_token.value
+        self.next_token()  # Move past the collection
+
+        # Now, parse the block of statements to execute
+        true_block = self.parse_for_block()
+
+        # Construct and return a ForNode
+        return ForNode(iterator, collection, true_block)
+
+    def parse_for_block(self):
+        block = []
+        # Skip initial 'DO' if present
+        if self.current_token and self.current_token.value == "DO":
+            self.next_token()
+
+        while self.current_token and self.current_token.value not in ("ENDFOR",):
+            if self.current_token.value == ";":
+                # If a semicolon is encountered, skip it and move to the next token
+                self.next_token()
+                continue
+            # Add the current token to the block unless it's a 'DO' or ';'
+            if self.current_token.value != "DO":
+                block.append(self.current_token.value)
+            self.next_token()
+
+        # The loop exits when 'ENDFOR' is encountered; move past it for subsequent parsing
+        self.next_token()  # Skip 'ENDFOR'
+        return block
+
+    def parse_try_statement(self):
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "TRY":
+            raise SyntaxError("Expected TRY statement")
+        self.next_token()  # Skip 'TRY'
+
+        try_block = self.parse_try_block("EXCEPT")  # Parse the try block until 'EXCEPT'
+
+        # Now expecting 'EXCEPT' keyword
+        if not (self.current_token and self.current_token.value == "EXCEPT"):
+            raise SyntaxError("Expected 'EXCEPT' after try block")
+        self.next_token()  # Move past 'EXCEPT'
+
+        except_block = self.parse_try_block(
+            "ENDTRY"
+        )  # Parse the except block until 'ENDTRY'
+
+        # Ensure we are correctly positioned after 'ENDTRY'
+        if self.current_token and self.current_token.value != "ENDTRY":
+            raise SyntaxError("Expected 'ENDTRY' at the end of except block")
+        self.next_token()  # Move past 'ENDTRY' for subsequent parsing
+
+        return TryNode(try_block, except_block)
+
+    def parse_try_block(self, stop_keyword):
+        block = []
+        while self.current_token and self.current_token.value != stop_keyword:
+            if self.current_token.value == "DO":
+                self.next_token()  # Move past 'DO' to get to the action
+            elif self.current_token.value == ";":
+                self.next_token()  # Move past the semicolon
+                continue  # Skip adding ';' to the block
+            else:
+                block.append(self.current_token.value)  # Add the action to the block
+                self.next_token()
+
+        return block
+
+
+# "IF condition1 && condition2; DO action2; ELSE; DO action3; ENDIF;"
+# "FOR input_ IN collections; DO action(input_); ENDFOR;"
+# "TRY; DO action(); EXCEPT; DO action(input_); ENDTRY;"

lionagi/experimental/directive/schema.py

@@ -0,0 +1,36 @@
+class Node:
+    """Base class for all nodes in the abstract syntax tree (AST)."""
+
+    pass
+
+
+class IfNode(Node):
+    """Represents an 'IF' statement in the AST."""
+
+    def __init__(self, condition, true_block, false_block=None):
+        self.condition = condition
+        self.true_block = true_block
+        self.false_block = false_block
+
+
+class ForNode(Node):
+    """Represents a 'FOR' loop in the AST."""
+
+    def __init__(self, iterator, collection, block):
+        self.iterator = iterator
+        self.collection = collection
+        self.block = block
+
+
+class TryNode(Node):
+    """Represents a 'TRY-EXCEPT' block in the AST."""
+
+    def __init__(self, try_block, except_block):
+        self.try_block = try_block
+        self.except_block = except_block
+
+
+class ActionNode(Node):
+
+    def __init__(self, action) -> None:
+        self.action = action

lionagi/experimental/directive/template_/base_template.py

@@ -0,0 +1,63 @@
+from typing import Any, Dict
+import re
+
+from ..evaluator.base_evaluator import BaseEvaluator
+
+
+class BaseDirectiveTemplate:
+    """Enhanced base template class for processing templates with conditionals and loops."""
+
+    def __init__(self, template_str: str):
+        self.template_str = template_str
+        self.evaluator = BaseEvaluator()
+
+    def _render_conditionals(self, context: Dict[str, Any]) -> str:
+        """Processes conditional statements with improved logic and support for 'else'."""
+        pattern = re.compile(r"\{if (.*?)\}(.*?)\{else\}(.*?)\{endif\}", re.DOTALL)
+
+        def evaluate_condition(match):
+            condition, if_text, else_text = match.groups()
+            if self.evaluator.evaluate(condition, context):
+                return if_text
+            else:
+                return else_text
+
+        return pattern.sub(evaluate_condition, self.template_str)
+
+    def _render_loops(self, template: str, context: Dict[str, Any]) -> str:
+        """Processes loop statements within the template."""
+        loop_pattern = re.compile(r"\{for (\w+) in (\w+)\}(.*?)\{endfor\}", re.DOTALL)
+
+        def render_loop(match):
+            iterator_var, collection_name, loop_body = match.groups()
+            collection = context.get(collection_name, [])
+            if not isinstance(collection, (list, range)):
+                raise ValueError(
+                    f"Expected list or range for '{collection_name}', got {type(collection).__name__}."
+                )
+
+            loop_result = ""
+            for item in collection:
+                loop_context = context.copy()
+                loop_context[iterator_var] = item
+                loop_result += self.fill(loop_body, loop_context)
+
+            return loop_result
+
+        return loop_pattern.sub(render_loop, template)
+
+    def fill(self, template_str: str = "", context: Dict[str, Any] = {}) -> str:
+        """Fills the template with values from context after processing conditionals and loops."""
+        if not template_str:  # Use the instance's template if not provided
+            template_str = self.template_str
+
+        # First, process conditionals with 'else'
+        template_with_conditionals = self._render_conditionals(template_str)
+        # Then, process loops
+        template_with_loops = self._render_loops(template_with_conditionals, context)
+        # Finally, substitute the placeholders with context values
+        try:
+            return template_with_loops.format(**context)
+        except KeyError as e:
+            print(f"Missing key in context: {e}")
+            return template_with_loops

lionagi/experimental/tool/function_calling.py

@@ -0,0 +1,43 @@
+from typing import Any, Callable
+from pydantic import BaseModel, Field, field_serializer
+from functools import singledispatchmethod
+from lionagi.libs import convert
+
+
+class FunctionCalling(BaseModel):
+    func: Any = Field(..., alias="function")
+    kwargs: Any = Field({}, alias="arguments")
+
+    @field_serializer("func")
+    def serialize_func(self, func: Callable):
+        return func.__name__
+
+    @property
+    def func_name(self):
+        return self.func.__name__
+
+    @classmethod
+    @singledispatchmethod
+    def create(cls, func_call: Any):
+        raise TypeError(f"Unsupported type {type(func_call)}")
+
+    @create.register
+    def _(cls, func_call: tuple):
+        if len(func_call) == 2:
+            return cls(func=func_call[0], kwargs=func_call[1])
+        else:
+            raise ValueError(f"Invalid tuple length {len(func_call)}")
+
+    @create.register
+    def _(cls, func_call: dict):
+        return cls(**func_call)
+
+    @create.register
+    def _(cls, func_call: str):
+        try:
+            return cls(**convert.to_dict(func_call))
+        except Exception as e:
+            raise ValueError(f"Invalid string {func_call}") from e
+
+    def __str__(self):
+        return f"{self.func_name}({self.kwargs})"