lionagi 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

lionagi/integrations/storage/storage_util.py

@@ -1,302 +0,0 @@
-import inspect
-import json
-import re
-
-from lionagi.core.action import DirectiveSelection, Tool, func_to_tool
-from lionagi.core.agent.base_agent import BaseAgent
-from lionagi.core.generic.edge_condition import EdgeCondition
-from lionagi.core.message import Instruction, RoledMessage, System
-
-
-def output_node_list(structure):
-    """
-    Processes a structure object to extract and format all associated nodes into a summary list and detailed output dictionary.
-
-    This function traverses a structure, extracting key properties of nodes and organizing them by type into a dictionary for easy access and manipulation.
-
-    Args:
-        structure: The structure object containing nodes and potentially other nested structures.
-
-    Returns:
-        tuple: A tuple containing a summary list of all nodes and a dictionary categorizing nodes by type.
-    """
-    summary_list = []
-    output = {}
-
-    structure_output = {
-        "ln_id": structure.ln_id,
-        "timestamp": structure.timestamp,
-        "type": structure.class_name,
-    }
-    summary_list.append(structure_output.copy())
-    structure_output["head_nodes"] = json.dumps(
-        [i.ln_id for i in structure.get_heads()]
-    )
-    # structure_output['nodes'] = json.dumps([i for i in structure.internal_nodes.keys()])
-    # structure_output['edges'] = json.dumps([i for i in structure.internal_edges.keys()])
-    output[structure_output["type"]] = [structure_output]
-    for node in structure.internal_nodes.values():
-        node_output = {
-            "ln_id": node.ln_id,
-            "timestamp": node.timestamp,
-            "type": node.class_name,
-        }
-        summary_list.append(node_output.copy())
-        if isinstance(node, System) or isinstance(node, Instruction):
-            node_output["content"] = json.dumps(node.content)
-            # node_output["sender"] = node.sender
-            # node_output["recipient"] = node.recipient
-        elif isinstance(node, Tool):
-            node_output["function"] = inspect.getsource(node.function)
-            # node_output['manual'] = node.manual
-            node_output["parser"] = (
-                inspect.getsource(node.parser) if node.parser else None
-            )
-        elif isinstance(node, DirectiveSelection):
-            node_output["directive"] = node.directive
-            node_output["directive_kwargs"] = json.dumps(node.directive_kwargs)
-        elif isinstance(node, BaseAgent):
-            node_output["structure_id"] = node.structure.ln_id
-            node_output["output_parser"] = (
-                inspect.getsource(node.output_parser)
-                if node.output_parser
-                else None
-            )
-        else:
-            raise ValueError("Not supported node type detected")
-        if node_output["type"] not in output:
-            output[node_output["type"]] = []
-        output[node_output["type"]].append(node_output)
-
-    return summary_list, output
-
-
-def output_edge_list(structure):
-    """
-    Extracts and formats all edges from a given structure into a list and maps any associated condition classes.
-
-    This function collects edge data from a structure, including identifiers, timestamps, labels, and conditions. It also compiles any unique condition classes associated with these edges.
-
-    Args:
-        structure: The structure object containing edges.
-
-    Returns:
-        tuple: A tuple containing a list of all edges with their details and a list of unique condition classes.
-    """
-    edge_list = []
-    edge_cls_dict = {}
-    for edge in structure.internal_edges.values():
-        edge_output = {
-            "ln_id": edge.ln_id,
-            "timestamp": edge.timestamp,
-            "head": edge.head,
-            "tail": edge.tail,
-            "label": edge.label,
-            "bundle": str(edge.bundle),
-        }
-        if edge.condition:
-            cls_name = edge.condition.__class__.__qualname__
-            cond = json.dumps(
-                {"class": cls_name, "args": edge.condition.model_dump()}
-            )
-            cls = edge.string_condition()
-            if cls is not None and cls_name not in edge_cls_dict:
-                edge_cls_dict.update({cls_name: cls})
-        else:
-            cond = None
-        edge_output["condition"] = cond
-        edge_list.append(edge_output)
-
-    edge_cls_list = []
-    for key, value in edge_cls_dict.items():
-        edge_cls_list.append({"class_name": key, "class": value})
-
-    return edge_list, edge_cls_list
-
-
-class ParseNode:
-    """
-    Provides static methods for converting code strings to functional definitions, classes, and for parsing various types of structured nodes based on dictionary definitions.
-
-    This utility class facilitates the dynamic execution of code and the instantiation of objects from serialized data.
-    """
-
-    @staticmethod
-    def convert_to_def(function_code):
-        """
-        Converts a string containing a function definition into a callable function object.
-
-        Args:
-            function_code (str): The string code of the function to convert.
-
-        Returns:
-            function: The converted function as a callable.
-
-        Raises:
-            ValueError: If the function code is invalid or the function name cannot be detected.
-        """
-        import re
-
-        match = re.search(r"def (\w+)\(", function_code)
-        if match:
-            class_name = match.group(1)
-            try:
-                exec(function_code, globals())
-                func = globals()[class_name]
-                return func
-            except Exception as e:
-                raise ValueError(f"Failed to convert str to def. Error: {e}")
-        else:
-            raise ValueError("Failed to detect function name")
-
-    @staticmethod
-    def convert_to_class(cls_code):
-        """
-        Converts a string containing a class definition into a class object.
-
-        Args:
-            cls_code (str): The string code of the class to convert.
-
-        Returns:
-            class: The converted class.
-
-        Raises:
-            ValueError: If the class code is invalid or the class name cannot be detected.
-        """
-        import re
-
-        match = re.search(r"class (\w+)\s*(?:\(([^)]+)\))?:", cls_code)
-        if match:
-            class_name = match.group(1)
-            try:
-                exec(cls_code, globals())
-                cls = globals()[class_name]
-                return cls
-            except Exception as e:
-                raise ValueError(f"Failed to convert str to class. Error: {e}")
-        else:
-            raise ValueError("Failed to detect class name")
-
-    @staticmethod
-    def parse_system(info_dict):
-        """
-        Parses dictionary information into a System node object.
-
-        Args:
-            info_dict (dict): A dictionary containing properties of a system node.
-
-        Returns:
-            System: An instantiated System node filled with properties from info_dict.
-        """
-        info_dict["system"] = json.loads(info_dict.pop("content"))[
-            "system_info"
-        ]
-        node = System.from_obj(info_dict)
-        return node
-
-    @staticmethod
-    def parse_instruction(info_dict):
-        """
-        Parses dictionary information into an Instruction node object.
-
-        Args:
-            info_dict (dict): A dictionary containing properties of an instruction node.
-
-        Returns:
-            Instruction: An instantiated Instruction node filled with properties from info_dict.
-        """
-        info_dict["instruction"] = json.loads(info_dict.pop("content"))[
-            "instruction"
-        ]
-        node = Instruction.from_obj(info_dict)
-        return node
-
-    @staticmethod
-    def parse_directiveSelection(info_dict):
-        """
-        Parses dictionary information into an DirectiveSelection node object.
-
-        Args:
-            info_dict (dict): A dictionary containing properties of an action selection node.
-
-        Returns:
-            ActionSelection: An instantiated ActionSelection node filled with properties from info_dict.
-        """
-        node = DirectiveSelection(ln_id=info_dict["ln_id"])
-        node.directive = info_dict["directive"]
-        if "directive_kwargs" in info_dict:
-            if info_dict["directive_kwargs"]:
-                node.directive_kwargs = json.loads(
-                    info_dict["directive_kwargs"]
-                )
-        elif "directiveKwargs" in info_dict:
-            if info_dict["directiveKwargs"]:
-                node.directive_kwargs = json.loads(
-                    info_dict["directiveKwargs"]
-                )
-        return node
-
-    @staticmethod
-    def parse_tool(info_dict):
-        """
-        Parses dictionary information into a Tool node object, converting associated function code into a callable.
-
-        Args:
-            info_dict (dict): A dictionary containing properties and function code for a tool node.
-
-        Returns:
-            Tool: An instantiated Tool node with the function converted from code.
-
-        Raises:
-            ValueError: If unsafe code is detected in the function definition.
-        """
-        func_code = info_dict["function"]
-        if (
-            "import os" in func_code
-            or "__" in func_code
-            or "import sys" in func_code
-        ):
-            raise ValueError(
-                "Unsafe code detected in Tool function. Please double check or implement explicitly"
-            )
-
-        func = ParseNode.convert_to_def(func_code)
-        tool = func_to_tool(
-            func, ln_id=info_dict["ln_id"], timestamp=info_dict["timestamp"]
-        )
-        if func.__doc__:
-            if re.search(r":param \w+:", func.__doc__):
-                tool = func_to_tool(
-                    func,
-                    docstring_style="reST",
-                    ln_id=info_dict["ln_id"],
-                    timestamp=info_dict["timestamp"],
-                )
-
-        tool = tool[0]
-        return tool
-
-    @staticmethod
-    def parse_condition(condition, cls_code):
-        """
-        Parses a condition dictionary and corresponding class code into a class instance representing the condition.
-
-        Args:
-            condition (dict): A dictionary containing the serialized form of the condition's arguments.
-            cls_code (str): The class code to instantiate the condition class.
-
-        Returns:
-            An instance of the condition class filled with properties from the condition dictionary.
-
-        Raises:
-            ValueError: If the condition or class code is invalid.
-        """
-        args = condition["args"]
-        cls = ParseNode.convert_to_class(cls_code)
-
-        init_params = {}
-        for key in inspect.signature(cls.__init__).parameters.keys():
-            if key == "self":
-                continue
-            init_params[key] = args[key]
-        return cls(**init_params)

lionagi/integrations/storage/structure_excel.py

@@ -1,291 +0,0 @@
-import json
-from pathlib import Path
-
-import pandas as pd
-
-from lionagi.core.agent.base_agent import BaseAgent
-from lionagi.core.engine.instruction_map_engine import InstructionMapEngine
-from lionagi.core.executor.base_executor import BaseExecutor
-from lionagi.core.executor.graph_executor import GraphExecutor
-from lionagi.integrations.storage.storage_util import ParseNode
-
-
-def excel_reload(
-    structure_name=None, structure_id=None, dir="structure_storage"
-):
-    """
-    Loads a structure from an Excel file into a StructureExecutor instance.
-
-    This function uses the StructureExcel class to handle the reloading process. It identifies the
-    Excel file based on the provided structure name or ID and reloads the structure from it.
-
-    Args:
-        structure_name (str, optional): The name of the structure to reload.
-        structure_id (str, optional): The unique identifier of the structure to reload.
-        dir (str): The directory path where the Excel files are stored.
-
-    Returns:
-        StructureExecutor: An instance of StructureExecutor containing the reloaded structure.
-
-    Raises:
-        ValueError: If neither structure_name nor structure_id is provided, or if multiple or no files
-                    are found matching the criteria.
-    """
-    excel_structure = StructureExcel(structure_name, structure_id, dir)
-    excel_structure.reload()
-    return excel_structure.structure
-
-
-class StructureExcel:
-    """
-    Manages the reloading of structures from Excel files.
-
-    This class handles the identification and parsing of structure data from an Excel workbook. It supports
-    loading from specifically named Excel files based on the structure name or ID.
-
-    Attributes:
-        structure (StructureExecutor): The loaded structure, ready for execution.
-        default_agent_executable (BaseExecutor): The default executor for agents within the structure.
-
-    Methods:
-        get_heads(): Retrieves the head nodes of the loaded structure.
-        parse_node(info_dict): Parses a node from the structure based on the provided dictionary.
-        get_next(node_id): Gets the next nodes connected by outgoing edges from a given node.
-        relate(parent_node, node): Relates two nodes within the structure based on the edge definitions.
-        parse(node_list, parent_node=None): Recursively parses nodes and their relationships from the structure.
-        reload(): Reloads the structure from the Excel file based on the initially provided parameters.
-    """
-
-    structure: GraphExecutor = GraphExecutor()
-    default_agent_executable: BaseExecutor = InstructionMapEngine()
-
-    def __init__(
-        self,
-        structure_name=None,
-        structure_id=None,
-        file_path="structure_storage",
-    ):
-        """
-        Initializes the StructureExcel class with specified parameters.
-
-        This method sets up the paths and reads the Excel file, preparing the internal dataframes used for
-        structure parsing.
-
-        Args:
-            structure_name (str, optional): The name of the structure to reload.
-            structure_id (str, optional): The unique identifier of the structure to reload.
-            file_path (str): The base path where the Excel files are stored.
-
-        Raises:
-            ValueError: If both structure_name and structure_id are provided but do not correspond to a valid file,
-                        or if multiple or no files are found when one of the identifiers is provided.
-        """
-        self.file_path = file_path
-        if not structure_name and not structure_id:
-            raise ValueError("Please provide the structure name or id")
-        if structure_name and structure_id:
-            self.filename = f"{file_path}/{structure_name}_{structure_id}.xlsx"
-            self.file = pd.read_excel(self.filename, sheet_name=None)
-        elif structure_name and not structure_id:
-            dir_path = Path(file_path)
-            files = list(dir_path.glob(f"{structure_name}*.xlsx"))
-            filename = []
-            for file in files:
-                try:
-                    name = file.name
-                    name = name.rsplit("_", 1)[0]
-                    if name == structure_name:
-                        filename.append(file.name)
-                except:
-                    continue
-            if len(filename) > 1:
-                raise ValueError(
-                    f"Multiple files starting with the same structure name {structure_name} has found, please specify the structure id"
-                )
-            self.filename = f"{file_path}/{filename[0]}"
-            self.file = pd.read_excel(self.filename, sheet_name=None)
-        elif structure_id and not structure_name:
-            dir_path = Path(file_path)
-            files = list(dir_path.glob(f"*{structure_id}.xlsx"))
-            filename = [file.name for file in files]
-            if len(filename) > 1:
-                raise ValueError(
-                    f"Multiple files with the same structure id {structure_id} has found, please double check the stored structure"
-                )
-            self.filename = f"{file_path}/{filename[0]}"
-            self.file = pd.read_excel(self.filename, sheet_name=None)
-        self.nodes = self.file["Nodes"]
-        self.edges = self.file["Edges"]
-
-    def get_heads(self):
-        """
-        Retrieves the list of head node identifiers from the loaded structure data.
-
-        This method parses the 'StructureExecutor' sheet in the loaded Excel file to extract the list of head nodes.
-
-        Returns:
-            list: A list of identifiers for the head nodes in the structure.
-        """
-        structure_df = self.file["GraphExecutor"]
-        head_list = json.loads(structure_df["head_nodes"].iloc[0])
-        return head_list
-
-    def _reload_info_dict(self, node_id):
-        """
-        Retrieves detailed information about a specific node from the Excel file based on its identifier.
-
-        This method looks up a node's information within the loaded Excel sheets and returns a dictionary
-        containing all the relevant details.
-
-        Args:
-            node_id (str): The identifier of the node to look up.
-
-        Returns:
-            dict: A dictionary containing the properties and values for the specified node.
-        """
-        node_type = self.nodes[self.nodes["ln_id"] == node_id]["type"].iloc[0]
-        node_file = self.file[node_type]
-        row = node_file[node_file["ln_id"] == node_id].iloc[0]
-        info_dict = row.to_dict()
-        return info_dict
-
-    def parse_agent(self, info_dict):
-        """
-        Parses an agent node from the structure using the agent's specific details provided in a dictionary.
-
-        This method creates an agent instance based on the information from the dictionary, which includes
-        dynamically loading the output parser code.
-
-        Args:
-            info_dict (dict): A dictionary containing details about the agent node, including its class, structure ID,
-                              and output parser code.
-
-        Returns:
-            BaseAgent: An initialized agent object.
-        """
-        output_parser = ParseNode.convert_to_def(info_dict["output_parser"])
-
-        structure_excel = StructureExcel(
-            structure_id=info_dict["structure_id"], file_path=self.file_path
-        )
-        structure_excel.reload()
-        structure = structure_excel.structure
-        agent = BaseAgent(
-            structure=structure,
-            executable=self.default_agent_executable,
-            output_parser=output_parser,
-            ln_id=info_dict["ln_id"],
-            timestamp=info_dict["timestamp"],
-        )
-        return agent
-
-    def parse_node(self, info_dict):
-        """
-        Parses a node from its dictionary representation into a specific node type like System, Instruction, etc.
-
-        This method determines the type of node from the info dictionary and uses the appropriate parsing method
-        to create an instance of that node type.
-
-        Args:
-            info_dict (dict): A dictionary containing node data, including the node type and associated properties.
-
-        Returns:
-            Node: An instance of the node corresponding to the type specified in the info_dict.
-        """
-        if info_dict["type"] == "System":
-            return ParseNode.parse_system(info_dict)
-        elif info_dict["type"] == "Instruction":
-            return ParseNode.parse_instruction(info_dict)
-        elif info_dict["type"] == "Tool":
-            return ParseNode.parse_tool(info_dict)
-        elif info_dict["type"] == "DirectiveSelection":
-            return ParseNode.parse_directiveSelection(info_dict)
-        elif info_dict["type"] == "BaseAgent":
-            return self.parse_agent(info_dict)
-
-    def get_next(self, node_id):
-        """
-        Retrieves the list of identifiers for nodes that are directly connected via outgoing edges from the specified node.
-
-        This method searches the 'Edges' DataFrame for all entries where the specified node is a head and returns
-        a list of the tail node identifiers.
-
-        Args:
-            node_id (str): The identifier of the node whose successors are to be found.
-
-        Returns:
-            list[str]: A list of identifiers for the successor nodes.
-        """
-        return self.edges[self.edges["head"] == node_id]["tail"].to_list()
-
-    def relate(self, parent_node, node):
-        """
-        Establishes a relationship between two nodes in the structure based on the Excel data for edges.
-
-        This method looks up the edge details connecting the two nodes and applies any conditions associated
-        with the edge to the structure being rebuilt.
-
-        Args:
-            parent_node (Node): The parent node in the relationship.
-            node (Node): The child node in the relationship.
-
-        Raises:
-            ValueError: If there are issues with the edge data such as multiple undefined edges.
-        """
-        if not parent_node:
-            return
-        row = self.edges[
-            (self.edges["head"] == parent_node.ln_id)
-            & (self.edges["tail"] == node.ln_id)
-        ]
-        if len(row) > 1:
-            raise ValueError(
-                f"currently does not support handle multiple edges between two nodes, Error node: from {parent_node.ln_id} to {node.ln_id}"
-            )
-        if row["condition"].isna().any():
-            self.structure.add_edge(parent_node, node)
-        else:
-            cond = json.loads(row["condition"].iloc[0])
-            cond_cls = cond["class"]
-            cond_row = self.file["EdgesCondClass"][
-                self.file["EdgesCondClass"]["class_name"] == cond_cls
-            ]
-            cond_code = cond_row["class"].iloc[0]
-            condition = ParseNode.parse_condition(cond, cond_code)
-            self.structure.add_edge(parent_node, node, condition=condition)
-
-    def parse(self, node_list, parent_node=None):
-        """
-        Recursively parses a list of nodes and establishes their interconnections based on the Excel data.
-
-        This method processes each node ID in the list, parsing individual nodes and relating them according
-        to their connections defined in the Excel file.
-
-        Args:
-            node_list (list[str]): A list of node identifiers to be parsed.
-            parent_node (Node, optional): The parent node to which the nodes in the list are connected.
-
-        Raises:
-            ValueError: If an error occurs during parsing or relating nodes.
-        """
-        for node_id in node_list:
-            info_dict = self._reload_info_dict(node_id)
-            node = self.parse_node(info_dict)
-
-            if node.ln_id not in self.structure.internal_nodes:
-                self.structure.add_node(node)
-            self.relate(parent_node, node)
-
-            next_node_list = self.get_next(node_id)
-            self.parse(next_node_list, node)
-
-    def reload(self):
-        """
-        Reloads the entire structure from the Excel file.
-
-        This method initializes a new StructureExecutor and uses the Excel data to rebuild the entire structure,
-        starting from the head nodes and recursively parsing and connecting all nodes defined within.
-        """
-        self.structure = GraphExecutor()
-        heads = self.get_heads()
-        self.parse(heads)

lionagi/integrations/storage/to_csv.py

@@ -1,70 +0,0 @@
-import zipfile
-from pathlib import Path
-
-import pandas as pd
-
-from lionagi.integrations.storage.storage_util import (
-    output_edge_list,
-    output_node_list,
-)
-
-
-def _output_csv(
-    node_list, node_dict, edge_list, edge_cls_list, zipname="structure_storage"
-):
-    """
-    Writes provided node and edge data into multiple CSV files and compresses them into a ZIP archive.
-
-    This helper function takes lists and dictionaries of nodes and edges, converts them to pandas DataFrames,
-    and then writes each DataFrame to a CSV file stored inside a ZIP archive. This includes a separate CSV
-    for each type of node and edge, as well as edge conditions if they exist.
-
-    Args:
-        node_list (list): A list of dictionaries where each dictionary contains attributes of a single node.
-        node_dict (dict): A dictionary of lists where each key represents a node type and the value is a list of
-                          node attributes for nodes of that type.
-        edge_list (list): A list of dictionaries where each dictionary contains attributes of a single edge.
-        edge_cls_list (list): A list of dictionaries where each dictionary contains attributes of edge conditions.
-        zipname (str): The base name for the output ZIP file that will store the CSV files.
-
-    Returns:
-        None: This function does not return a value but outputs a ZIP file containing the CSVs.
-    """
-    tables = {
-        "Nodes": pd.DataFrame(node_list),
-        "Edges": pd.DataFrame(edge_list),
-    }
-    if edge_cls_list:
-        tables["EdgesCondClass"] = pd.DataFrame(edge_cls_list)
-    for i in node_dict:
-        tables[i] = pd.DataFrame(node_dict[i])
-
-    zipname = zipname + ".zip"
-
-    with zipfile.ZipFile(zipname, "w") as zf:
-        for i in tables:
-            filename = i + ".csv"
-            with zf.open(filename, "w") as file:
-                tables[i].to_csv(file, index=False)
-
-
-def to_csv(structure, filename="structure_storage"):
-    """
-    Converts a structure into a series of CSV files and stores them in a compressed ZIP archive.
-
-    This function processes a given structure to extract detailed node and edge information,
-    including conditions if applicable. These details are then saved into separate CSV files
-    for nodes, edges, and any edge conditions, which are subsequently bundled into a ZIP file.
-
-    Args:
-        structure: An object representing the structure to be serialized. This structure should
-                   have methods to return lists of nodes and edges.
-        filename (str): The base name of the output ZIP file that will store the CSV files.
-
-    Returns:
-        None: This function does not return a value but outputs a ZIP file containing CSVs.
-    """
-    node_list, node_dict = output_node_list(structure)
-    edge_list, edge_cls_list = output_edge_list(structure)
-
-    _output_csv(node_list, node_dict, edge_list, edge_cls_list, filename)