lionagi 0.0.312__py3-none-any.whl → 0.2.1__py3-none-any.whl

lionagi/integrations/storage/storage_util.py

@@ -0,0 +1,287 @@
+import json
+import inspect
+import re
+
+from lionagi.core.message import System, Instruction, RoledMessage
+from lionagi.core.action import Tool, DirectiveSelection, func_to_tool
+from lionagi.core.action import DirectiveSelection
+from lionagi.core.agent.base_agent import BaseAgent
+from lionagi.core.generic.edge_condition import EdgeCondition
+
+
+def output_node_list(structure):
+    """
+    Processes a structure object to extract and format all associated nodes into a summary list and detailed output dictionary.
+
+    This function traverses a structure, extracting key properties of nodes and organizing them by type into a dictionary for easy access and manipulation.
+
+    Args:
+        structure: The structure object containing nodes and potentially other nested structures.
+
+    Returns:
+        tuple: A tuple containing a summary list of all nodes and a dictionary categorizing nodes by type.
+    """
+    summary_list = []
+    output = {}
+
+    structure_output = {
+        "ln_id": structure.ln_id,
+        "timestamp": structure.timestamp,
+        "type": structure.class_name,
+    }
+    summary_list.append(structure_output.copy())
+    structure_output["head_nodes"] = json.dumps(
+        [i.ln_id for i in structure.get_heads()]
+    )
+    # structure_output['nodes'] = json.dumps([i for i in structure.internal_nodes.keys()])
+    # structure_output['edges'] = json.dumps([i for i in structure.internal_edges.keys()])
+    output[structure_output["type"]] = [structure_output]
+    for node in structure.internal_nodes.values():
+        node_output = {
+            "ln_id": node.ln_id,
+            "timestamp": node.timestamp,
+            "type": node.class_name,
+        }
+        summary_list.append(node_output.copy())
+        if isinstance(node, System) or isinstance(node, Instruction):
+            node_output["content"] = json.dumps(node.content)
+            # node_output["sender"] = node.sender
+            # node_output["recipient"] = node.recipient
+        elif isinstance(node, Tool):
+            node_output["function"] = inspect.getsource(node.function)
+            # node_output['manual'] = node.manual
+            node_output["parser"] = (
+                inspect.getsource(node.parser) if node.parser else None
+            )
+        elif isinstance(node, DirectiveSelection):
+            node_output["directive"] = node.directive
+            node_output["directive_kwargs"] = json.dumps(node.directive_kwargs)
+        elif isinstance(node, BaseAgent):
+            node_output["structure_id"] = node.structure.ln_id
+            node_output["output_parser"] = (
+                inspect.getsource(node.output_parser) if node.output_parser else None
+            )
+        else:
+            raise ValueError("Not supported node type detected")
+        if node_output["type"] not in output:
+            output[node_output["type"]] = []
+        output[node_output["type"]].append(node_output)
+
+    return summary_list, output
+
+
+def output_edge_list(structure):
+    """
+    Extracts and formats all edges from a given structure into a list and maps any associated condition classes.
+
+    This function collects edge data from a structure, including identifiers, timestamps, labels, and conditions. It also compiles any unique condition classes associated with these edges.
+
+    Args:
+        structure: The structure object containing edges.
+
+    Returns:
+        tuple: A tuple containing a list of all edges with their details and a list of unique condition classes.
+    """
+    edge_list = []
+    edge_cls_dict = {}
+    for edge in structure.internal_edges.values():
+        edge_output = {
+            "ln_id": edge.ln_id,
+            "timestamp": edge.timestamp,
+            "head": edge.head,
+            "tail": edge.tail,
+            "label": edge.label,
+            "bundle": str(edge.bundle),
+        }
+        if edge.condition:
+            cls_name = edge.condition.__class__.__qualname__
+            cond = json.dumps({"class": cls_name, "args": edge.condition.model_dump()})
+            cls = edge.string_condition()
+            if cls is not None and cls_name not in edge_cls_dict:
+                edge_cls_dict.update({cls_name: cls})
+        else:
+            cond = None
+        edge_output["condition"] = cond
+        edge_list.append(edge_output)
+
+    edge_cls_list = []
+    for key, value in edge_cls_dict.items():
+        edge_cls_list.append({"class_name": key, "class": value})
+
+    return edge_list, edge_cls_list
+
+
+class ParseNode:
+    """
+    Provides static methods for converting code strings to functional definitions, classes, and for parsing various types of structured nodes based on dictionary definitions.
+
+    This utility class facilitates the dynamic execution of code and the instantiation of objects from serialized data.
+    """
+
+    @staticmethod
+    def convert_to_def(function_code):
+        """
+        Converts a string containing a function definition into a callable function object.
+
+        Args:
+            function_code (str): The string code of the function to convert.
+
+        Returns:
+            function: The converted function as a callable.
+
+        Raises:
+            ValueError: If the function code is invalid or the function name cannot be detected.
+        """
+        import re
+
+        match = re.search(r"def (\w+)\(", function_code)
+        if match:
+            class_name = match.group(1)
+            try:
+                exec(function_code, globals())
+                func = globals()[class_name]
+                return func
+            except Exception as e:
+                raise ValueError(f"Failed to convert str to def. Error: {e}")
+        else:
+            raise ValueError("Failed to detect function name")
+
+    @staticmethod
+    def convert_to_class(cls_code):
+        """
+        Converts a string containing a class definition into a class object.
+
+        Args:
+            cls_code (str): The string code of the class to convert.
+
+        Returns:
+            class: The converted class.
+
+        Raises:
+            ValueError: If the class code is invalid or the class name cannot be detected.
+        """
+        import re
+
+        match = re.search(r"class (\w+)\s*(?:\(([^)]+)\))?:", cls_code)
+        if match:
+            class_name = match.group(1)
+            try:
+                exec(cls_code, globals())
+                cls = globals()[class_name]
+                return cls
+            except Exception as e:
+                raise ValueError(f"Failed to convert str to class. Error: {e}")
+        else:
+            raise ValueError("Failed to detect class name")
+
+    @staticmethod
+    def parse_system(info_dict):
+        """
+        Parses dictionary information into a System node object.
+
+        Args:
+            info_dict (dict): A dictionary containing properties of a system node.
+
+        Returns:
+            System: An instantiated System node filled with properties from info_dict.
+        """
+        info_dict["system"] = json.loads(info_dict.pop("content"))["system_info"]
+        node = System.from_obj(info_dict)
+        return node
+
+    @staticmethod
+    def parse_instruction(info_dict):
+        """
+        Parses dictionary information into an Instruction node object.
+
+        Args:
+            info_dict (dict): A dictionary containing properties of an instruction node.
+
+        Returns:
+            Instruction: An instantiated Instruction node filled with properties from info_dict.
+        """
+        info_dict["instruction"] = json.loads(info_dict.pop("content"))["instruction"]
+        node = Instruction.from_obj(info_dict)
+        return node
+
+    @staticmethod
+    def parse_directiveSelection(info_dict):
+        """
+        Parses dictionary information into an DirectiveSelection node object.
+
+        Args:
+            info_dict (dict): A dictionary containing properties of an action selection node.
+
+        Returns:
+            ActionSelection: An instantiated ActionSelection node filled with properties from info_dict.
+        """
+        node = DirectiveSelection(ln_id=info_dict["ln_id"])
+        node.directive = info_dict["directive"]
+        if "directive_kwargs" in info_dict:
+            if info_dict["directive_kwargs"]:
+                node.directive_kwargs = json.loads(info_dict["directive_kwargs"])
+        elif "directiveKwargs" in info_dict:
+            if info_dict["directiveKwargs"]:
+                node.directive_kwargs = json.loads(info_dict["directiveKwargs"])
+        return node
+
+    @staticmethod
+    def parse_tool(info_dict):
+        """
+        Parses dictionary information into a Tool node object, converting associated function code into a callable.
+
+        Args:
+            info_dict (dict): A dictionary containing properties and function code for a tool node.
+
+        Returns:
+            Tool: An instantiated Tool node with the function converted from code.
+
+        Raises:
+            ValueError: If unsafe code is detected in the function definition.
+        """
+        func_code = info_dict["function"]
+        if "import os" in func_code or "__" in func_code or "import sys" in func_code:
+            raise ValueError(
+                "Unsafe code detected in Tool function. Please double check or implement explicitly"
+            )
+
+        func = ParseNode.convert_to_def(func_code)
+        tool = func_to_tool(
+            func, ln_id=info_dict["ln_id"], timestamp=info_dict["timestamp"]
+        )
+        if func.__doc__:
+            if re.search(r":param \w+:", func.__doc__):
+                tool = func_to_tool(
+                    func,
+                    docstring_style="reST",
+                    ln_id=info_dict["ln_id"],
+                    timestamp=info_dict["timestamp"],
+                )
+
+        tool = tool[0]
+        return tool
+
+    @staticmethod
+    def parse_condition(condition, cls_code):
+        """
+        Parses a condition dictionary and corresponding class code into a class instance representing the condition.
+
+        Args:
+            condition (dict): A dictionary containing the serialized form of the condition's arguments.
+            cls_code (str): The class code to instantiate the condition class.
+
+        Returns:
+            An instance of the condition class filled with properties from the condition dictionary.
+
+        Raises:
+            ValueError: If the condition or class code is invalid.
+        """
+        args = condition["args"]
+        cls = ParseNode.convert_to_class(cls_code)
+
+        init_params = {}
+        for key in inspect.signature(cls.__init__).parameters.keys():
+            if key == "self":
+                continue
+            init_params[key] = args[key]
+        return cls(**init_params)

lionagi/integrations/storage/structure_excel.py

@@ -0,0 +1,285 @@
+import pandas as pd
+import json
+from pathlib import Path
+
+from lionagi.integrations.storage.storage_util import ParseNode
+from lionagi.core.executor.graph_executor import GraphExecutor
+from lionagi.core.agent.base_agent import BaseAgent
+from lionagi.core.executor.base_executor import BaseExecutor
+from lionagi.core.engine.instruction_map_engine import InstructionMapEngine
+
+
+def excel_reload(structure_name=None, structure_id=None, dir="structure_storage"):
+    """
+    Loads a structure from an Excel file into a StructureExecutor instance.
+
+    This function uses the StructureExcel class to handle the reloading process. It identifies the
+    Excel file based on the provided structure name or ID and reloads the structure from it.
+
+    Args:
+        structure_name (str, optional): The name of the structure to reload.
+        structure_id (str, optional): The unique identifier of the structure to reload.
+        dir (str): The directory path where the Excel files are stored.
+
+    Returns:
+        StructureExecutor: An instance of StructureExecutor containing the reloaded structure.
+
+    Raises:
+        ValueError: If neither structure_name nor structure_id is provided, or if multiple or no files
+                    are found matching the criteria.
+    """
+    excel_structure = StructureExcel(structure_name, structure_id, dir)
+    excel_structure.reload()
+    return excel_structure.structure
+
+
+class StructureExcel:
+    """
+    Manages the reloading of structures from Excel files.
+
+    This class handles the identification and parsing of structure data from an Excel workbook. It supports
+    loading from specifically named Excel files based on the structure name or ID.
+
+    Attributes:
+        structure (StructureExecutor): The loaded structure, ready for execution.
+        default_agent_executable (BaseExecutor): The default executor for agents within the structure.
+
+    Methods:
+        get_heads(): Retrieves the head nodes of the loaded structure.
+        parse_node(info_dict): Parses a node from the structure based on the provided dictionary.
+        get_next(node_id): Gets the next nodes connected by outgoing edges from a given node.
+        relate(parent_node, node): Relates two nodes within the structure based on the edge definitions.
+        parse(node_list, parent_node=None): Recursively parses nodes and their relationships from the structure.
+        reload(): Reloads the structure from the Excel file based on the initially provided parameters.
+    """
+
+    structure: GraphExecutor = GraphExecutor()
+    default_agent_executable: BaseExecutor = InstructionMapEngine()
+
+    def __init__(
+        self, structure_name=None, structure_id=None, file_path="structure_storage"
+    ):
+        """
+        Initializes the StructureExcel class with specified parameters.
+
+        This method sets up the paths and reads the Excel file, preparing the internal dataframes used for
+        structure parsing.
+
+        Args:
+            structure_name (str, optional): The name of the structure to reload.
+            structure_id (str, optional): The unique identifier of the structure to reload.
+            file_path (str): The base path where the Excel files are stored.
+
+        Raises:
+            ValueError: If both structure_name and structure_id are provided but do not correspond to a valid file,
+                        or if multiple or no files are found when one of the identifiers is provided.
+        """
+        self.file_path = file_path
+        if not structure_name and not structure_id:
+            raise ValueError("Please provide the structure name or id")
+        if structure_name and structure_id:
+            self.filename = f"{file_path}/{structure_name}_{structure_id}.xlsx"
+            self.file = pd.read_excel(self.filename, sheet_name=None)
+        elif structure_name and not structure_id:
+            dir_path = Path(file_path)
+            files = list(dir_path.glob(f"{structure_name}*.xlsx"))
+            filename = []
+            for file in files:
+                try:
+                    name = file.name
+                    name = name.rsplit("_", 1)[0]
+                    if name == structure_name:
+                        filename.append(file.name)
+                except:
+                    continue
+            if len(filename) > 1:
+                raise ValueError(
+                    f"Multiple files starting with the same structure name {structure_name} has found, please specify the structure id"
+                )
+            self.filename = f"{file_path}/{filename[0]}"
+            self.file = pd.read_excel(self.filename, sheet_name=None)
+        elif structure_id and not structure_name:
+            dir_path = Path(file_path)
+            files = list(dir_path.glob(f"*{structure_id}.xlsx"))
+            filename = [file.name for file in files]
+            if len(filename) > 1:
+                raise ValueError(
+                    f"Multiple files with the same structure id {structure_id} has found, please double check the stored structure"
+                )
+            self.filename = f"{file_path}/{filename[0]}"
+            self.file = pd.read_excel(self.filename, sheet_name=None)
+        self.nodes = self.file["Nodes"]
+        self.edges = self.file["Edges"]
+
+    def get_heads(self):
+        """
+        Retrieves the list of head node identifiers from the loaded structure data.
+
+        This method parses the 'StructureExecutor' sheet in the loaded Excel file to extract the list of head nodes.
+
+        Returns:
+            list: A list of identifiers for the head nodes in the structure.
+        """
+        structure_df = self.file["GraphExecutor"]
+        head_list = json.loads(structure_df["head_nodes"].iloc[0])
+        return head_list
+
+    def _reload_info_dict(self, node_id):
+        """
+        Retrieves detailed information about a specific node from the Excel file based on its identifier.
+
+        This method looks up a node's information within the loaded Excel sheets and returns a dictionary
+        containing all the relevant details.
+
+        Args:
+            node_id (str): The identifier of the node to look up.
+
+        Returns:
+            dict: A dictionary containing the properties and values for the specified node.
+        """
+        node_type = self.nodes[self.nodes["ln_id"] == node_id]["type"].iloc[0]
+        node_file = self.file[node_type]
+        row = node_file[node_file["ln_id"] == node_id].iloc[0]
+        info_dict = row.to_dict()
+        return info_dict
+
+    def parse_agent(self, info_dict):
+        """
+        Parses an agent node from the structure using the agent's specific details provided in a dictionary.
+
+        This method creates an agent instance based on the information from the dictionary, which includes
+        dynamically loading the output parser code.
+
+        Args:
+            info_dict (dict): A dictionary containing details about the agent node, including its class, structure ID,
+                              and output parser code.
+
+        Returns:
+            BaseAgent: An initialized agent object.
+        """
+        output_parser = ParseNode.convert_to_def(info_dict["output_parser"])
+
+        structure_excel = StructureExcel(
+            structure_id=info_dict["structure_id"], file_path=self.file_path
+        )
+        structure_excel.reload()
+        structure = structure_excel.structure
+        agent = BaseAgent(
+            structure=structure,
+            executable=self.default_agent_executable,
+            output_parser=output_parser,
+            ln_id=info_dict["ln_id"],
+            timestamp=info_dict["timestamp"],
+        )
+        return agent
+
+    def parse_node(self, info_dict):
+        """
+        Parses a node from its dictionary representation into a specific node type like System, Instruction, etc.
+
+        This method determines the type of node from the info dictionary and uses the appropriate parsing method
+        to create an instance of that node type.
+
+        Args:
+            info_dict (dict): A dictionary containing node data, including the node type and associated properties.
+
+        Returns:
+            Node: An instance of the node corresponding to the type specified in the info_dict.
+        """
+        if info_dict["type"] == "System":
+            return ParseNode.parse_system(info_dict)
+        elif info_dict["type"] == "Instruction":
+            return ParseNode.parse_instruction(info_dict)
+        elif info_dict["type"] == "Tool":
+            return ParseNode.parse_tool(info_dict)
+        elif info_dict["type"] == "DirectiveSelection":
+            return ParseNode.parse_directiveSelection(info_dict)
+        elif info_dict["type"] == "BaseAgent":
+            return self.parse_agent(info_dict)
+
+    def get_next(self, node_id):
+        """
+        Retrieves the list of identifiers for nodes that are directly connected via outgoing edges from the specified node.
+
+        This method searches the 'Edges' DataFrame for all entries where the specified node is a head and returns
+        a list of the tail node identifiers.
+
+        Args:
+            node_id (str): The identifier of the node whose successors are to be found.
+
+        Returns:
+            list[str]: A list of identifiers for the successor nodes.
+        """
+        return self.edges[self.edges["head"] == node_id]["tail"].to_list()
+
+    def relate(self, parent_node, node):
+        """
+        Establishes a relationship between two nodes in the structure based on the Excel data for edges.
+
+        This method looks up the edge details connecting the two nodes and applies any conditions associated
+        with the edge to the structure being rebuilt.
+
+        Args:
+            parent_node (Node): The parent node in the relationship.
+            node (Node): The child node in the relationship.
+
+        Raises:
+            ValueError: If there are issues with the edge data such as multiple undefined edges.
+        """
+        if not parent_node:
+            return
+        row = self.edges[
+            (self.edges["head"] == parent_node.ln_id)
+            & (self.edges["tail"] == node.ln_id)
+        ]
+        if len(row) > 1:
+            raise ValueError(
+                f"currently does not support handle multiple edges between two nodes, Error node: from {parent_node.ln_id} to {node.ln_id}"
+            )
+        if row["condition"].isna().any():
+            self.structure.add_edge(parent_node, node)
+        else:
+            cond = json.loads(row["condition"].iloc[0])
+            cond_cls = cond["class"]
+            cond_row = self.file["EdgesCondClass"][
+                self.file["EdgesCondClass"]["class_name"] == cond_cls
+            ]
+            cond_code = cond_row["class"].iloc[0]
+            condition = ParseNode.parse_condition(cond, cond_code)
+            self.structure.add_edge(parent_node, node, condition=condition)
+
+    def parse(self, node_list, parent_node=None):
+        """
+        Recursively parses a list of nodes and establishes their interconnections based on the Excel data.
+
+        This method processes each node ID in the list, parsing individual nodes and relating them according
+        to their connections defined in the Excel file.
+
+        Args:
+            node_list (list[str]): A list of node identifiers to be parsed.
+            parent_node (Node, optional): The parent node to which the nodes in the list are connected.
+
+        Raises:
+            ValueError: If an error occurs during parsing or relating nodes.
+        """
+        for node_id in node_list:
+            info_dict = self._reload_info_dict(node_id)
+            node = self.parse_node(info_dict)
+
+            if node.ln_id not in self.structure.internal_nodes:
+                self.structure.add_node(node)
+            self.relate(parent_node, node)
+
+            next_node_list = self.get_next(node_id)
+            self.parse(next_node_list, node)
+
+    def reload(self):
+        """
+        Reloads the entire structure from the Excel file.
+
+        This method initializes a new StructureExecutor and uses the Excel data to rebuild the entire structure,
+        starting from the head nodes and recursively parsing and connecting all nodes defined within.
+        """
+        self.structure = GraphExecutor()
+        heads = self.get_heads()
+        self.parse(heads)

lionagi/integrations/storage/to_csv.py

@@ -0,0 +1,63 @@
+import zipfile
+import pandas as pd
+from pathlib import Path
+
+from lionagi.integrations.storage.storage_util import output_node_list, output_edge_list
+
+
+def _output_csv(
+    node_list, node_dict, edge_list, edge_cls_list, zipname="structure_storage"
+):
+    """
+    Writes provided node and edge data into multiple CSV files and compresses them into a ZIP archive.
+
+    This helper function takes lists and dictionaries of nodes and edges, converts them to pandas DataFrames,
+    and then writes each DataFrame to a CSV file stored inside a ZIP archive. This includes a separate CSV
+    for each type of node and edge, as well as edge conditions if they exist.
+
+    Args:
+        node_list (list): A list of dictionaries where each dictionary contains attributes of a single node.
+        node_dict (dict): A dictionary of lists where each key represents a node type and the value is a list of
+                          node attributes for nodes of that type.
+        edge_list (list): A list of dictionaries where each dictionary contains attributes of a single edge.
+        edge_cls_list (list): A list of dictionaries where each dictionary contains attributes of edge conditions.
+        zipname (str): The base name for the output ZIP file that will store the CSV files.
+
+    Returns:
+        None: This function does not return a value but outputs a ZIP file containing the CSVs.
+    """
+    tables = {"Nodes": pd.DataFrame(node_list), "Edges": pd.DataFrame(edge_list)}
+    if edge_cls_list:
+        tables["EdgesCondClass"] = pd.DataFrame(edge_cls_list)
+    for i in node_dict:
+        tables[i] = pd.DataFrame(node_dict[i])
+
+    zipname = zipname + ".zip"
+
+    with zipfile.ZipFile(zipname, "w") as zf:
+        for i in tables:
+            filename = i + ".csv"
+            with zf.open(filename, "w") as file:
+                tables[i].to_csv(file, index=False)
+
+
+def to_csv(structure, filename="structure_storage"):
+    """
+    Converts a structure into a series of CSV files and stores them in a compressed ZIP archive.
+
+    This function processes a given structure to extract detailed node and edge information,
+    including conditions if applicable. These details are then saved into separate CSV files
+    for nodes, edges, and any edge conditions, which are subsequently bundled into a ZIP file.
+
+    Args:
+        structure: An object representing the structure to be serialized. This structure should
+                   have methods to return lists of nodes and edges.
+        filename (str): The base name of the output ZIP file that will store the CSV files.
+
+    Returns:
+        None: This function does not return a value but outputs a ZIP file containing CSVs.
+    """
+    node_list, node_dict = output_node_list(structure)
+    edge_list, edge_cls_list = output_edge_list(structure)
+
+    _output_csv(node_list, node_dict, edge_list, edge_cls_list, filename)