lionagi 0.0.316__py3-none-any.whl → 0.1.1__py3-none-any.whl

lionagi/integrations/storage/storage_util.py

@@ -0,0 +1,289 @@
+import json
+import inspect
+import re
+
+from lionagi.core import System, Instruction
+from lionagi.core.tool import Tool
+from lionagi.core.generic.action import ActionSelection
+from lionagi.core.agent.base_agent import BaseAgent
+from lionagi.core.generic.condition import Condition
+
+from lionagi.core import func_to_tool
+
+
+def output_node_list(structure):
+    """
+    Processes a structure object to extract and format all associated nodes into a summary list and detailed output dictionary.
+
+    This function traverses a structure, extracting key properties of nodes and organizing them by type into a dictionary for easy access and manipulation.
+
+    Args:
+        structure: The structure object containing nodes and potentially other nested structures.
+
+    Returns:
+        tuple: A tuple containing a summary list of all nodes and a dictionary categorizing nodes by type.
+    """
+    summary_list = []
+    output = {}
+
+    structure_output = {
+        "id": structure.id_,
+        "timestamp": structure.timestamp,
+        "type": structure.class_name(),
+    }
+    summary_list.append(structure_output.copy())
+    structure_output["head_nodes"] = [i.id_ for i in structure.get_heads()]
+    # structure_output['nodes'] = json.dumps([i for i in structure.internal_nodes.keys()])
+    # structure_output['edges'] = json.dumps([i for i in structure.internal_edges.keys()])
+    output[structure_output["type"]] = [structure_output]
+    for node in structure.internal_nodes.values():
+        node_output = {
+            "id": node.id_,
+            "timestamp": node.timestamp,
+            "type": node.class_name(),
+        }
+        summary_list.append(node_output.copy())
+        if isinstance(node, System) or isinstance(node, Instruction):
+            node_output["content"] = json.dumps(node.content)
+            node_output["sender"] = node.sender
+            node_output["recipient"] = node.recipient
+        elif isinstance(node, Tool):
+            node_output["function"] = inspect.getsource(node.func)
+            # node_output['manual'] = node.manual
+            node_output["parser"] = (
+                inspect.getsource(node.parser) if node.parser else None
+            )
+        elif isinstance(node, ActionSelection):
+            node_output["action"] = node.action
+            node_output["action_kwargs"] = json.dumps(node.action_kwargs)
+        elif isinstance(node, BaseAgent):
+            node_output["structure_id"] = node.structure.id_
+            node_output["output_parser"] = inspect.getsource(node.output_parser)
+        else:
+            raise ValueError("Not supported node type detected")
+        if node_output["type"] not in output:
+            output[node_output["type"]] = []
+        output[node_output["type"]].append(node_output)
+
+    return summary_list, output
+
+
+def output_edge_list(structure):
+    """
+    Extracts and formats all edges from a given structure into a list and maps any associated condition classes.
+
+    This function collects edge data from a structure, including identifiers, timestamps, labels, and conditions. It also compiles any unique condition classes associated with these edges.
+
+    Args:
+        structure: The structure object containing edges.
+
+    Returns:
+        tuple: A tuple containing a list of all edges with their details and a list of unique condition classes.
+    """
+    edge_list = []
+    edge_cls_dict = {}
+    for edge in structure.internal_edges.values():
+        edge_output = {
+            "id": edge.id_,
+            "timestamp": edge.timestamp,
+            "head": edge.head,
+            "tail": edge.tail,
+            "label": edge.label,
+            "bundle": str(edge.bundle),
+        }
+        if edge.condition:
+            cls_name = edge.condition.__class__.__qualname__
+            cond = json.dumps({"class": cls_name, "args": edge.condition.model_dump()})
+            cls = edge.string_condition()
+            if cls is not None and cls_name not in edge_cls_dict:
+                edge_cls_dict.update({cls_name: cls})
+        else:
+            cond = None
+        edge_output["condition"] = cond
+        edge_list.append(edge_output)
+
+    edge_cls_list = []
+    for key, value in edge_cls_dict.items():
+        edge_cls_list.append({"class_name": key, "class": value})
+
+    return edge_list, edge_cls_list
+
+
+class ParseNode:
+    """
+    Provides static methods for converting code strings to functional definitions, classes, and for parsing various types of structured nodes based on dictionary definitions.
+
+    This utility class facilitates the dynamic execution of code and the instantiation of objects from serialized data.
+    """
+
+    @staticmethod
+    def convert_to_def(function_code):
+        """
+        Converts a string containing a function definition into a callable function object.
+
+        Args:
+            function_code (str): The string code of the function to convert.
+
+        Returns:
+            function: The converted function as a callable.
+
+        Raises:
+            ValueError: If the function code is invalid or the function name cannot be detected.
+        """
+        import re
+
+        match = re.search(r"def (\w+)\(", function_code)
+        if match:
+            class_name = match.group(1)
+            try:
+                exec(function_code, globals())
+                func = globals()[class_name]
+                return func
+            except Exception as e:
+                raise ValueError(f"Failed to convert str to def. Error: {e}")
+        else:
+            raise ValueError("Failed to detect function name")
+
+    @staticmethod
+    def convert_to_class(cls_code):
+        """
+        Converts a string containing a class definition into a class object.
+
+        Args:
+            cls_code (str): The string code of the class to convert.
+
+        Returns:
+            class: The converted class.
+
+        Raises:
+            ValueError: If the class code is invalid or the class name cannot be detected.
+        """
+        import re
+
+        match = re.search(r"class (\w+)\s*(?:\(([^)]+)\))?:", cls_code)
+        if match:
+            class_name = match.group(1)
+            try:
+                exec(cls_code, globals())
+                cls = globals()[class_name]
+                return cls
+            except Exception as e:
+                raise ValueError(f"Failed to convert str to class. Error: {e}")
+        else:
+            raise ValueError("Failed to detect class name")
+
+    @staticmethod
+    def parse_system(info_dict):
+        """
+        Parses dictionary information into a System node object.
+
+        Args:
+            info_dict (dict): A dictionary containing properties of a system node.
+
+        Returns:
+            System: An instantiated System node filled with properties from info_dict.
+        """
+        node = System(" ")
+        node.id_ = info_dict["id"]
+        node.timestamp = info_dict["timestamp"]
+        node.content = json.loads(info_dict["content"])
+        node.sender = info_dict["sender"]
+        node.recipient = info_dict["recipient"]
+        return node
+
+    @staticmethod
+    def parse_instruction(info_dict):
+        """
+        Parses dictionary information into an Instruction node object.
+
+        Args:
+            info_dict (dict): A dictionary containing properties of an instruction node.
+
+        Returns:
+            Instruction: An instantiated Instruction node filled with properties from info_dict.
+        """
+        node = Instruction(" ")
+        node.id_ = info_dict["id"]
+        node.timestamp = info_dict["timestamp"]
+        node.content = json.loads(info_dict["content"])
+        node.sender = info_dict["sender"]
+        node.recipient = info_dict["recipient"]
+        return node
+
+    @staticmethod
+    def parse_actionSelection(info_dict):
+        """
+        Parses dictionary information into an ActionSelection node object.
+
+        Args:
+            info_dict (dict): A dictionary containing properties of an action selection node.
+
+        Returns:
+            ActionSelection: An instantiated ActionSelection node filled with properties from info_dict.
+        """
+        node = ActionSelection()
+        node.id_ = info_dict["id"]
+        node.action = info_dict["action"]
+        if "action_kwargs" in info_dict:
+            if info_dict["action_kwargs"]:
+                node.action_kwargs = json.loads(info_dict["action_kwargs"])
+        elif "actionKwargs" in info_dict:
+            if info_dict["actionKwargs"]:
+                node.action_kwargs = json.loads(info_dict["actionKwargs"])
+        return node
+
+    @staticmethod
+    def parse_tool(info_dict):
+        """
+        Parses dictionary information into a Tool node object, converting associated function code into a callable.
+
+        Args:
+            info_dict (dict): A dictionary containing properties and function code for a tool node.
+
+        Returns:
+            Tool: An instantiated Tool node with the function converted from code.
+
+        Raises:
+            ValueError: If unsafe code is detected in the function definition.
+        """
+        func_code = info_dict["function"]
+        if "import os" in func_code or "__" in func_code or "import sys" in func_code:
+            raise ValueError(
+                "Unsafe code detected in Tool function. Please double check or implement explicitly"
+            )
+
+        func = ParseNode.convert_to_def(func_code)
+        tool = func_to_tool(func)
+        if func.__doc__:
+            if re.search(r":param \w+:", func.__doc__):
+                tool = func_to_tool(func, docstring_style="reST")
+
+        tool = tool[0]
+        tool.id_ = info_dict["id"]
+        tool.timestamp = info_dict["timestamp"]
+        return tool
+
+    @staticmethod
+    def parse_condition(condition, cls_code):
+        """
+        Parses a condition dictionary and corresponding class code into a class instance representing the condition.
+
+        Args:
+            condition (dict): A dictionary containing the serialized form of the condition's arguments.
+            cls_code (str): The class code to instantiate the condition class.
+
+        Returns:
+            An instance of the condition class filled with properties from the condition dictionary.
+
+        Raises:
+            ValueError: If the condition or class code is invalid.
+        """
+        args = condition["args"]
+        cls = ParseNode.convert_to_class(cls_code)
+
+        init_params = {}
+        for key in inspect.signature(cls.__init__).parameters.keys():
+            if key == "self":
+                continue
+            init_params[key] = args[key]
+        return cls(**init_params)

lionagi/integrations/storage/to_csv.py

@@ -0,0 +1,63 @@
+import zipfile
+import pandas as pd
+from pathlib import Path
+
+from lionagi.integrations.storage.storage_util import output_node_list, output_edge_list
+
+
+def _output_csv(
+    node_list, node_dict, edge_list, edge_cls_list, zipname="structure_storage"
+):
+    """
+    Writes provided node and edge data into multiple CSV files and compresses them into a ZIP archive.
+
+    This helper function takes lists and dictionaries of nodes and edges, converts them to pandas DataFrames,
+    and then writes each DataFrame to a CSV file stored inside a ZIP archive. This includes a separate CSV
+    for each type of node and edge, as well as edge conditions if they exist.
+
+    Args:
+        node_list (list): A list of dictionaries where each dictionary contains attributes of a single node.
+        node_dict (dict): A dictionary of lists where each key represents a node type and the value is a list of
+                          node attributes for nodes of that type.
+        edge_list (list): A list of dictionaries where each dictionary contains attributes of a single edge.
+        edge_cls_list (list): A list of dictionaries where each dictionary contains attributes of edge conditions.
+        zipname (str): The base name for the output ZIP file that will store the CSV files.
+
+    Returns:
+        None: This function does not return a value but outputs a ZIP file containing the CSVs.
+    """
+    tables = {"Nodes": pd.DataFrame(node_list), "Edges": pd.DataFrame(edge_list)}
+    if edge_cls_list:
+        tables["EdgesCondClass"] = pd.DataFrame(edge_cls_list)
+    for i in node_dict:
+        tables[i] = pd.DataFrame(node_dict[i])
+
+    zipname = zipname + ".zip"
+
+    with zipfile.ZipFile(zipname, "w") as zf:
+        for i in tables:
+            filename = i + ".csv"
+            with zf.open(filename, "w") as file:
+                tables[i].to_csv(file, index=False)
+
+
+def to_csv(structure, filename="structure_storage"):
+    """
+    Converts a structure into a series of CSV files and stores them in a compressed ZIP archive.
+
+    This function processes a given structure to extract detailed node and edge information,
+    including conditions if applicable. These details are then saved into separate CSV files
+    for nodes, edges, and any edge conditions, which are subsequently bundled into a ZIP file.
+
+    Args:
+        structure: An object representing the structure to be serialized. This structure should
+                   have methods to return lists of nodes and edges.
+        filename (str): The base name of the output ZIP file that will store the CSV files.
+
+    Returns:
+        None: This function does not return a value but outputs a ZIP file containing CSVs.
+    """
+    node_list, node_dict = output_node_list(structure)
+    edge_list, edge_cls_list = output_edge_list(structure)
+
+    _output_csv(node_list, node_dict, edge_list, edge_cls_list, filename)

lionagi/integrations/storage/to_excel.py

@@ -0,0 +1,67 @@
+import pandas as pd
+from lionagi.libs import SysUtil
+
+from lionagi.integrations.storage.storage_util import output_node_list, output_edge_list
+
+
+def _output_excel(
+    node_list, node_dict, edge_list, edge_cls_list, filename="structure_storage"
+):
+    """
+    Writes provided node and edge data into multiple sheets of a single Excel workbook.
+
+    This helper function takes lists and dictionaries of nodes and edges, converts them into pandas DataFrames,
+    and then writes each DataFrame to a distinct sheet in an Excel workbook. This includes a separate sheet
+    for each type of node and edge, as well as edge conditions if they exist.
+
+    Args:
+        node_list (list): A list of dictionaries where each dictionary contains attributes of a single node.
+        node_dict (dict): A dictionary of lists where each key represents a node type and the value is a list of
+                          node attributes for nodes of that type.
+        edge_list (list): A list of dictionaries where each dictionary contains attributes of a single edge.
+        edge_cls_list (list): A list of dictionaries where each dictionary contains attributes of edge conditions.
+        filename (str): The base name for the output Excel file. The '.xlsx' extension will be added
+                        automatically if not included.
+
+    Returns:
+        None: This function does not return a value but writes data directly to an Excel workbook.
+
+    Raises:
+        ImportError: If the required 'openpyxl' library is not installed, which is necessary for pandas to write Excel files.
+    """
+    SysUtil.check_import("openpyxl")
+
+    tables = {"Nodes": pd.DataFrame(node_list), "Edges": pd.DataFrame(edge_list)}
+    if edge_cls_list:
+        tables["EdgesCondClass"] = pd.DataFrame(edge_cls_list)
+    for i in node_dict:
+        tables[i] = pd.DataFrame(node_dict[i])
+
+    filename = filename + ".xlsx"
+
+    with pd.ExcelWriter(filename) as writer:
+        for i in tables:
+            tables[i].to_excel(writer, sheet_name=i, index=False)
+
+
+def to_excel(structure, filename="structure_storage"):
+    """
+    Converts a structure into a series of Excel sheets within a single workbook.
+
+    This function processes the specified structure to extract detailed node and edge information,
+    including conditions if applicable. These details are then saved into separate sheets within an
+    Excel workbook for nodes, edges, and any edge conditions.
+
+    Args:
+        structure: An object representing the structure to be serialized. This should have methods
+                   to return lists of nodes and edges suitable for output.
+        filename (str): The base name of the output Excel file. The '.xlsx' extension will be added
+                        automatically if not included.
+
+    Returns:
+        None: This function does not return a value but outputs an Excel workbook with multiple sheets.
+    """
+    node_list, node_dict = output_node_list(structure)
+    edge_list, edge_cls_list = output_edge_list(structure)
+
+    _output_excel(node_list, node_dict, edge_list, edge_cls_list, filename)

lionagi/libs/ln_api.py

@@ -377,8 +377,8 @@ class APIUtil:
                 num_tokens += 2  # every reply is primed with <im_start>assistant
                 return num_tokens + completion_tokens
             else:
-                prompt = payload["prompt"]
-                if isinstance(prompt, str):  # single prompt
+                prompt = payload["format_prompt"]
+                if isinstance(prompt, str):  # single format_prompt
                     prompt_tokens = len(encoding.encode(prompt))
                     return prompt_tokens + completion_tokens
                 elif isinstance(prompt, list):  # multiple prompts
@@ -386,7 +386,7 @@ class APIUtil:
                     return prompt_tokens + completion_tokens * len(prompt)
                 else:
                     raise TypeError(
-                        'Expecting either string or list of strings for "prompt" field in completion request'
+                        'Expecting either string or list of strings for "format_prompt" field in completion request'
                     )
         elif api_endpoint == "embeddings":
             input = payload["input"]