lionagi 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

lionagi/experimental/directive/parser/base_parser.py

@@ -0,0 +1,215 @@
+from typing import List, Optional
+
+from lionagi.libs.ln_tokenizer import BaseToken
+from ..schema import IfNode, TryNode, ForNode
+
+
+class BaseDirectiveParser:
+    """A base parser with lookahead, error recovery, and backtracking support.
+
+    Attributes:
+        tokens (List[BaseToken]): A list of tokens to be parsed.
+        current_token_index (int): The index of the current token in the tokens list.
+        current_token (Optional[BaseToken]): The current token being processed.
+
+    Examples:
+        >>> tokenizer = BaseTokenizer("IF x > 10 THEN DO something ENDIF")
+        >>> tokens = tokenizer.get_tokens()
+        >>> parser = BaseParser(tokens)
+        >>> print(parser.current_token)
+        BaseToken(KEYWORD, IF)
+    """
+
+    def __init__(self, tokens: List[BaseToken]):
+        self.tokens = tokens
+        self.current_token_index = -1
+        self.current_token: Optional[BaseToken] = None
+        self.next_token()
+
+    def next_token(self) -> None:
+        """Advances to the next token in the list."""
+        self.current_token_index += 1
+        if self.current_token_index < len(self.tokens):
+            self.current_token = self.tokens[self.current_token_index]
+        else:
+            self.current_token = None
+
+    def peek_next_token(self, offset: int = 1) -> BaseToken | None:
+        """Peeks at the next token without consuming it.
+
+        Args:
+            offset (int): The number of tokens to look ahead.
+
+        Returns:
+            Optional[BaseToken]: The token at the specified lookahead offset, or None if end of list.
+        """
+        peek_index = self.current_token_index + offset
+        if peek_index < len(self.tokens):
+            return self.tokens[peek_index]
+        else:
+            return None
+
+    def skip_until(self, token_types: List[str]) -> None:
+        """Skips tokens until a token of the specified type is found.
+
+        Args:
+            token_types (List[str]): A list of token types to stop skipping.
+        """
+        while self.current_token and self.current_token.type not in token_types:
+            self.next_token()
+
+    def mark(self) -> int:
+        """Marks the current position in the token list for potential backtracking.
+
+        Returns:
+            int: The current token index.
+        """
+        return self.current_token_index
+
+    def reset_to_mark(self, mark: int) -> None:
+        """Resets the parser to a previously marked position.
+
+        Args:
+            mark (int): The token index to reset to.
+        """
+        self.current_token_index = mark - 1
+        self.next_token()
+
+    def skip_semicolon(self):
+        if self.current_token and self.current_token.value == ";":
+            self.next_token()
+
+    def parse_expression(self):
+        expr = ""
+        while self.current_token and self.current_token.value != ";":
+            expr += self.current_token.value + " "
+            self.next_token()
+        # Expecting a semicolon at the end of the condition
+        if self.current_token.value != ";":
+            raise SyntaxError("Expected ';' at the end of the condition")
+        self.next_token()  # Move past the semicolon to the next part of the statement
+        return expr.strip()
+
+    def parse_if_block(self):
+        block = []
+        # Parse the block until 'ELSE', 'ENDIF', ensuring not to include semicolons as part of the block
+        while self.current_token and self.current_token.value not in ("ENDIF", "ELSE"):
+            if self.current_token.value == "DO":
+                self.next_token()  # Move past 'DO' to get to the action
+            block.append(self.current_token.value)  # Add the action to the block
+            self.next_token()  # Move to the next token, which could be a semicolon or the next action
+            if self.current_token.value == ";":
+                self.next_token()  # Move past the semicolon
+        return block
+
+    def parse_if_statement(self):
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "IF":
+            raise SyntaxError("Expected IF statement")
+        self.next_token()  # Skip 'IF'
+
+        condition = self.parse_expression()  # Now properly ends after the semicolon
+
+        true_block = []
+        if self.current_token.value == "DO":
+            true_block = self.parse_if_block()  # Parse true block after 'DO'
+
+        false_block = None
+        if self.current_token and self.current_token.value == "ELSE":
+            self.next_token()  # Skip 'ELSE', expect 'DO' next for the false block
+            self.skip_semicolon()
+            if self.current_token.value != "DO":
+                raise SyntaxError("Expected 'DO' after 'ELSE'")
+            self.next_token()  # Skip 'DO'
+            false_block = self.parse_if_block()  # Parse false block
+
+        return IfNode(condition, true_block, false_block)
+
+    def parse_for_statement(self):
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "FOR":
+            raise SyntaxError("Expected FOR statement")
+        self.next_token()  # Skip 'FOR'
+
+        # Parse the iterator variable
+        if self.current_token.type != "IDENTIFIER":
+            raise SyntaxError("Expected iterator variable after FOR")
+        iterator = self.current_token.value
+        self.next_token()  # Move past the iterator variable
+
+        # Expect and skip 'IN' keyword
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "IN":
+            raise SyntaxError("Expected 'IN' after iterator variable")
+        self.next_token()  # Move past 'IN'
+
+        # Parse the collection
+        if self.current_token.type not in ["IDENTIFIER", "LITERAL"]:
+            raise SyntaxError("Expected collection after 'IN'")
+        collection = self.current_token.value
+        self.next_token()  # Move past the collection
+
+        # Now, parse the block of statements to execute
+        true_block = self.parse_for_block()
+
+        # Construct and return a ForNode
+        return ForNode(iterator, collection, true_block)
+
+    def parse_for_block(self):
+        block = []
+        # Skip initial 'DO' if present
+        if self.current_token and self.current_token.value == "DO":
+            self.next_token()
+
+        while self.current_token and self.current_token.value not in ("ENDFOR",):
+            if self.current_token.value == ";":
+                # If a semicolon is encountered, skip it and move to the next token
+                self.next_token()
+                continue
+            # Add the current token to the block unless it's a 'DO' or ';'
+            if self.current_token.value != "DO":
+                block.append(self.current_token.value)
+            self.next_token()
+
+        # The loop exits when 'ENDFOR' is encountered; move past it for subsequent parsing
+        self.next_token()  # Skip 'ENDFOR'
+        return block
+
+    def parse_try_statement(self):
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "TRY":
+            raise SyntaxError("Expected TRY statement")
+        self.next_token()  # Skip 'TRY'
+
+        try_block = self.parse_try_block("EXCEPT")  # Parse the try block until 'EXCEPT'
+
+        # Now expecting 'EXCEPT' keyword
+        if not (self.current_token and self.current_token.value == "EXCEPT"):
+            raise SyntaxError("Expected 'EXCEPT' after try block")
+        self.next_token()  # Move past 'EXCEPT'
+
+        except_block = self.parse_try_block(
+            "ENDTRY"
+        )  # Parse the except block until 'ENDTRY'
+
+        # Ensure we are correctly positioned after 'ENDTRY'
+        if self.current_token and self.current_token.value != "ENDTRY":
+            raise SyntaxError("Expected 'ENDTRY' at the end of except block")
+        self.next_token()  # Move past 'ENDTRY' for subsequent parsing
+
+        return TryNode(try_block, except_block)
+
+    def parse_try_block(self, stop_keyword):
+        block = []
+        while self.current_token and self.current_token.value != stop_keyword:
+            if self.current_token.value == "DO":
+                self.next_token()  # Move past 'DO' to get to the action
+            elif self.current_token.value == ";":
+                self.next_token()  # Move past the semicolon
+                continue  # Skip adding ';' to the block
+            else:
+                block.append(self.current_token.value)  # Add the action to the block
+                self.next_token()
+
+        return block
+
+
+# "IF condition1 && condition2; DO action2; ELSE; DO action3; ENDIF;"
+# "FOR input_ IN collections; DO action(input_); ENDFOR;"
+# "TRY; DO action(); EXCEPT; DO action(input_); ENDTRY;"

lionagi/experimental/directive/schema.py

@@ -0,0 +1,36 @@
+class Node:
+    """Base class for all nodes in the abstract syntax tree (AST)."""
+
+    pass
+
+
+class IfNode(Node):
+    """Represents an 'IF' statement in the AST."""
+
+    def __init__(self, condition, true_block, false_block=None):
+        self.condition = condition
+        self.true_block = true_block
+        self.false_block = false_block
+
+
+class ForNode(Node):
+    """Represents a 'FOR' loop in the AST."""
+
+    def __init__(self, iterator, collection, block):
+        self.iterator = iterator
+        self.collection = collection
+        self.block = block
+
+
+class TryNode(Node):
+    """Represents a 'TRY-EXCEPT' block in the AST."""
+
+    def __init__(self, try_block, except_block):
+        self.try_block = try_block
+        self.except_block = except_block
+
+
+class ActionNode(Node):
+
+    def __init__(self, action) -> None:
+        self.action = action

lionagi/experimental/directive/template_/base_template.py

@@ -0,0 +1,63 @@
+from typing import Any, Dict
+import re
+
+from ..evaluator.base_evaluator import BaseEvaluator
+
+
+class BaseDirectiveTemplate:
+    """Enhanced base template class for processing templates with conditionals and loops."""
+
+    def __init__(self, template_str: str):
+        self.template_str = template_str
+        self.evaluator = BaseEvaluator()
+
+    def _render_conditionals(self, context: Dict[str, Any]) -> str:
+        """Processes conditional statements with improved logic and support for 'else'."""
+        pattern = re.compile(r"\{if (.*?)\}(.*?)\{else\}(.*?)\{endif\}", re.DOTALL)
+
+        def evaluate_condition(match):
+            condition, if_text, else_text = match.groups()
+            if self.evaluator.evaluate(condition, context):
+                return if_text
+            else:
+                return else_text
+
+        return pattern.sub(evaluate_condition, self.template_str)
+
+    def _render_loops(self, template: str, context: Dict[str, Any]) -> str:
+        """Processes loop statements within the template."""
+        loop_pattern = re.compile(r"\{for (\w+) in (\w+)\}(.*?)\{endfor\}", re.DOTALL)
+
+        def render_loop(match):
+            iterator_var, collection_name, loop_body = match.groups()
+            collection = context.get(collection_name, [])
+            if not isinstance(collection, (list, range)):
+                raise ValueError(
+                    f"Expected list or range for '{collection_name}', got {type(collection).__name__}."
+                )
+
+            loop_result = ""
+            for item in collection:
+                loop_context = context.copy()
+                loop_context[iterator_var] = item
+                loop_result += self.fill(loop_body, loop_context)
+
+            return loop_result
+
+        return loop_pattern.sub(render_loop, template)
+
+    def fill(self, template_str: str = "", context: Dict[str, Any] = {}) -> str:
+        """Fills the template with values from context after processing conditionals and loops."""
+        if not template_str:  # Use the instance's template if not provided
+            template_str = self.template_str
+
+        # First, process conditionals with 'else'
+        template_with_conditionals = self._render_conditionals(template_str)
+        # Then, process loops
+        template_with_loops = self._render_loops(template_with_conditionals, context)
+        # Finally, substitute the placeholders with context values
+        try:
+            return template_with_loops.format(**context)
+        except KeyError as e:
+            print(f"Missing key in context: {e}")
+            return template_with_loops

lionagi/experimental/report/form.py

@@ -0,0 +1,64 @@
+from pydantic import Field
+
+# from lionagi import logging as _logging
+from lionagi.core.generic import BaseComponent
+from lionagi.experimental.report.util import get_input_output_fields, system_fields
+
+
+class Form(BaseComponent):
+
+    assignment: str = Field(..., examples=["input1, input2 -> output"])
+
+    input_fields: list[str] = Field(default_factory=list)
+    output_fields: list[str] = Field(default_factory=list)
+
+    def __init__(self, **kwargs):
+        """
+        at initialization, all relevant fields if not already provided, are set to None,
+        not every field is required to be filled, nor required to be declared at initialization
+        """
+        super().__init__(**kwargs)
+        self.input_fields, self.output_fields = get_input_output_fields(self.assignment)
+        for i in self.input_fields + self.output_fields:
+            if i not in self.model_fields:
+                self._add_field(i, value=None)
+
+    @property
+    def workable(self):
+        if self.filled:
+            return False
+
+        for i in self.input_fields:
+            if not getattr(self, i, None):
+                return False
+
+        return True
+
+    @property
+    def work_fields(self):
+        dict_ = self.to_dict()
+        return {
+            k: v
+            for k, v in dict_.items()
+            if k not in system_fields and k in self.input_fields + self.output_fields
+        }
+
+    @property
+    def filled(self):
+        return all([value is not None for _, value in self.work_fields.items()])
+
+    def fill(self, form: "Form" = None, **kwargs):
+        """
+        only work fields for this form can be filled
+        a field can only be filled once
+        """
+        if self.filled:
+            raise ValueError("Form is already filled")
+
+        fields = form.work_fields if form else {}
+        kwargs = {**fields, **kwargs}
+
+        for k, v in kwargs.items():
+            if k not in self.work_fields:
+                raise ValueError(f"Field {k} is not a valid work field")
+            setattr(self, k, v)

lionagi/experimental/report/report.py

@@ -0,0 +1,138 @@
+from typing import Any, Type
+from pydantic import Field
+
+# from lionagi import logging as _logging
+from lionagi.core.generic import BaseComponent
+from lionagi.experimental.report.form import Form
+from lionagi.experimental.report.util import get_input_output_fields
+
+
+class Report(BaseComponent):
+
+    assignment: str = Field(..., examples=["input1, input2 -> output"])
+
+    forms: dict[str, Form] = Field(
+        default_factory=dict,
+        description="A dictionary of forms related to the report, in {assignment: Form} format.",
+    )
+
+    form_assignments: list = Field(
+        [],
+        description="assignment for the report",
+        examples=[["a, b -> c", "a -> e", "b -> f", "c -> g", "e, f, g -> h"]],
+    )
+
+    form_template: Type[Form] = Field(
+        Form, description="The template for the forms in the report."
+    )
+
+    input_fields: list[str] = Field(default_factory=list)
+    output_fields: list[str] = Field(default_factory=list)
+
+    def __init__(self, **kwargs):
+        """
+        at initialization, all relevant fields if not already provided, are set to None
+        """
+        super().__init__(**kwargs)
+        self.input_fields, self.output_fields = get_input_output_fields(self.assignment)
+
+        # if assignments is not provided, set it to assignment
+        if self.form_assignments == []:
+            self.form_assignments.append(self.assignment)
+
+        # create forms
+        new_forms = {i: self.form_template(assignment=i) for i in self.form_assignments}
+
+        # add new forms into the report (will ignore new forms already in the
+        # report with same assignment)
+        for k, v in new_forms.items():
+            if k not in self.forms:
+                self.forms[k] = v
+
+        # if the fields are not declared in the report, add them to report
+        # with value set to None
+        for k, v in self.forms.items():
+            for f in list(v.work_fields.keys()):
+                if f not in self.model_fields:
+                    field = v.model_fields[f]
+                    self._add_field(f, value=None, field=field)
+
+        # if there are fields in the report that are not in the forms, add them to
+        # the forms with values
+        for k, v in self.model_fields.items():
+            if getattr(self, k, None) is not None:
+                for f in self.forms.values():
+                    if k in f.work_fields:
+                        f.fill(**{k: getattr(self, k)})
+
+    @property
+    def work_fields(self) -> dict[str, Any]:
+        """
+        all work fields across all forms, including intermediate output fields,
+        this information is extracted from the forms
+        """
+
+        all_fields = {}
+        for form in self.forms.values():
+            for k, v in form.work_fields.items():
+                if k not in all_fields:
+                    all_fields[k] = v
+        return all_fields
+
+    def fill(self, **kwargs):
+        """
+        fill the information to both the report and forms
+        """
+        kwargs = {**self.work_fields, **kwargs}
+        for k, v in kwargs.items():
+            if k in self.work_fields and getattr(self, k, None) is None:
+                setattr(self, k, v)
+
+        for form in self.forms.values():
+            if not form.filled:
+                _kwargs = {k: v for k, v in kwargs.items() if k in form.work_fields}
+                form.fill(**_kwargs)
+
+    @property
+    def filled(self):
+        return all([value is not None for _, value in self.work_fields.items()])
+
+    @property
+    def workable(self) -> bool:
+
+        if self.filled:
+            # _logging.info("The report is already filled, no need to work on it.")
+            return False
+
+        for i in self.input_fields:
+            if not getattr(self, i, None):
+                # _logging.error(f"Field '{i}' is required to work on the report.")
+                return False
+
+        # this is the required fields from report's own assignment
+        fields = self.input_fields
+        fields.extend(self.output_fields)
+
+        # if the report's own assignment is not in the forms, return False
+        for f in fields:
+            if f not in self.work_fields:
+                # _logging.error(f"Field {f} is a required deliverable, not found in work field.")
+                return False
+
+        # get all the output fields from all the forms
+        outs = []
+        for form in self.forms.values():
+            outs.extend(form.output_fields)
+
+        # all output fields should be unique, not a single output field should be
+        # calculated by more than one form
+        if len(outs) != len(set(outs)):
+            # _logging.error("There are duplicate output fields in the forms.")
+            return False
+
+        return True
+
+    @property
+    def next_forms(self) -> list[Form] | None:
+        a = [i for i in self.forms.values() if i.workable]
+        return a if len(a) > 0 else None

lionagi/experimental/report/util.py

@@ -0,0 +1,47 @@
+from lionagi.libs import convert
+
+system_fields = [
+    "id_",
+    "node_id",
+    "meta",
+    "metadata",
+    "timestamp",
+    "content",
+    "assignment",
+    "assignments",
+    "task",
+    "template_name",
+    "version",
+    "description",
+    "in_validation_kwargs",
+    "out_validation_kwargs",
+    "fix_input",
+    "fix_output",
+    "input_fields",
+    "output_fields",
+    "choices",
+    "prompt_fields",
+    "prompt_fields_annotation",
+    "instruction_context",
+    "instruction",
+    "instruction_output_fields",
+    "inputs",
+    "outputs",
+    "process",
+    "_validate_field",
+    "_process_input",
+    "_process_response",
+    "_validate_field_choices",
+    "_validate_input_choices",
+    "_validate_output_choices",
+]
+
+
+def get_input_output_fields(str_: str) -> list[list[str]]:
+
+    inputs, outputs = str_.split("->")
+
+    input_fields = [convert.strip_lower(i) for i in inputs.split(",")]
+    output_fields = [convert.strip_lower(o) for o in outputs.split(",")]
+
+    return input_fields, output_fields

lionagi/experimental/tool/function_calling.py

@@ -0,0 +1,43 @@
+from typing import Any, Callable
+from pydantic import BaseModel, Field, field_serializer
+from functools import singledispatchmethod
+from lionagi.libs import convert
+
+
+class FunctionCalling(BaseModel):
+    func: Any = Field(..., alias="function")
+    kwargs: Any = Field({}, alias="arguments")
+
+    @field_serializer("func")
+    def serialize_func(self, func: Callable):
+        return func.__name__
+
+    @property
+    def func_name(self):
+        return self.func.__name__
+
+    @classmethod
+    @singledispatchmethod
+    def create(cls, func_call: Any):
+        raise TypeError(f"Unsupported type {type(func_call)}")
+
+    @create.register
+    def _(cls, func_call: tuple):
+        if len(func_call) == 2:
+            return cls(func=func_call[0], kwargs=func_call[1])
+        else:
+            raise ValueError(f"Invalid tuple length {len(func_call)}")
+
+    @create.register
+    def _(cls, func_call: dict):
+        return cls(**func_call)
+
+    @create.register
+    def _(cls, func_call: str):
+        try:
+            return cls(**convert.to_dict(func_call))
+        except Exception as e:
+            raise ValueError(f"Invalid string {func_call}") from e
+
+    def __str__(self):
+        return f"{self.func_name}({self.kwargs})"

lionagi/experimental/tool/manual.py

@@ -0,0 +1,66 @@
+import re
+from typing import Dict, Union, Callable, Any
+
+
+class BaseManual:
+    def __init__(self, template_str: str):
+        self.template_str = template_str
+
+    def _evaluate_condition(self, match, context):
+        condition, text = match.groups()
+        # Future implementations might parse and evaluate the condition more thoroughly
+        return text if condition in context and context[condition] else ""
+
+    def _render_conditionals(self, context: Dict[str, Union[str, int, float]]) -> str:
+        conditional_pattern = re.compile(r"\{if (.*?)\}(.*?)\{endif\}", re.DOTALL)
+        return conditional_pattern.sub(
+            lambda match: self._evaluate_condition(match, context), self.template_str
+        )
+
+    def _replace_callable(self, match, context):
+        key = match.group(1)
+        if key in context:
+            value = context[key]
+            return str(value() if callable(value) else value)
+        return match.group(0)  # Unmatched placeholders remain unchanged.
+
+    def _render_placeholders(
+        self,
+        rendered_template: str,
+        context: Dict[str, Union[str, int, float, Callable]],
+    ) -> str:
+        return re.sub(
+            r"\{(\w+)\}",
+            lambda match: self._replace_callable(match, context),
+            rendered_template,
+        )
+
+    def generate(self, context: Dict[str, Union[str, int, float, Callable]]) -> str:
+        """
+        Generates output by first processing conditionals, then rendering placeholders,
+        including executing callable objects for dynamic data generation.
+        """
+        template_with_conditionals = self._render_conditionals(context)
+        final_output = self._render_placeholders(template_with_conditionals, context)
+        return final_output
+
+
+# from experiments.executor.executor import SafeEvaluator
+
+# class DecisionTreeManual:
+#     def __init__(self, root):
+#         self.root = root
+#         self.evaluator = SafeEvaluator()
+
+#     def evaluate(self, context):
+#         return self._traverse_tree(self.root, context)
+
+#     def _traverse_tree(self, node, context):
+#         if isinstance(node, CompositeActionNode) or isinstance(node, ActionNode):
+#             return node.execute(context)
+#         elif isinstance(node, DecisionNode):
+#             condition_result = self.evaluator.evaluate(node.condition, context)
+#             next_node = node.true_branch if condition_result else node.false_branch
+#             return self._traverse_tree(next_node, context)
+#         else:
+#             raise ValueError("Invalid node type.")