lionagi 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

lionagi/experimental/work2/work_function.py

@@ -0,0 +1,89 @@
+import asyncio
+from typing import Any, Callable, Dict, List
+from pydantic import Field
+from functools import wraps
+from lionagi import logging as _logging
+from lionagi.libs import func_call
+from lionagi.core.generic import BaseComponent
+
+from .schema import Work, WorkStatus
+from ..work.worklog import WorkLog
+from .worker import Worker
+
+
+class WorkFunction(BaseComponent):
+    """Work function management and execution."""
+
+    function: Callable
+    args: List[Any] = Field(default_factory=list)
+    kwargs: Dict[str, Any] = Field(default_factory=dict)
+    retry_kwargs: Dict[str, Any] = Field(default_factory=dict)
+    worklog: WorkLog = Field(default_factory=WorkLog)
+    instruction: str = Field(
+        default="", description="Instruction for the work function"
+    )
+    refresh_time: float = Field(
+        default=0.5, description="Time to wait before checking for pending work"
+    )
+
+    @property
+    def name(self):
+        """Get the name of the work function."""
+        return self.function.__name__
+
+    async def execute(self):
+        """Execute pending work items."""
+        while self.worklog.pending:
+            work_id = self.worklog.pending.popleft()
+            work = self.worklog.logs[work_id]
+            if work.status == WorkStatus.PENDING:
+                try:
+                    await func_call.rcall(self._execute, work, **work.retry_kwargs)
+                except Exception as e:
+                    work.status = WorkStatus.FAILED
+                    _logging.error(f"Work {work.id_} failed with error: {e}")
+                    self.worklog.errored.append(work.id_)
+            else:
+                _logging.warning(
+                    f"Work {work.id_} is in {work.status} state "
+                    "and cannot be executed."
+                )
+            await asyncio.sleep(self.refresh_time)
+
+    async def _execute(self, work: Work):
+        """Execute a single work item."""
+        work.status = WorkStatus.IN_PROGRESS
+        result = await self.function(*self.args, **self.kwargs)
+        work.deliverables = result
+        work.status = WorkStatus.COMPLETED
+        return result
+
+
+def workfunc(func):
+
+    @wraps(func)
+    async def wrapper(self: Worker, *args, **kwargs):
+        # Retrieve the worker instance ('self')
+        if not hasattr(self, "work_functions"):
+            self.work_functions = {}
+
+        if func.__name__ not in self.work_functions:
+            # Create WorkFunction with the function and its docstring as instruction
+            self.work_functions[func.__name__] = WorkFunction(
+                function=func,
+                instruction=func.__doc__,
+                args=args,
+                kwargs=kwargs,
+                retry_kwargs=kwargs.pop("retry_kwargs", {}),
+            )
+
+        # Retrieve the existing WorkFunction
+        work_function: WorkFunction = self.work_functions[func.__name__]
+        # Update args and kwargs for this call
+        work_function.args = args
+        work_function.kwargs = kwargs
+
+        # Execute the function using WorkFunction's managed execution process
+        return await work_function.execute()
+
+    return wrapper

lionagi/experimental/work2/worker.py

@@ -0,0 +1,12 @@
+from abc import ABC
+from pydantic import Field
+from lionagi.core.generic import BaseComponent
+
+
+class Worker(BaseComponent, ABC):
+    form_templates: dict = Field(
+        default={}, description="The form templates of the worker"
+    )
+    work_functions: dict = Field(
+        default={}, description="The work functions of the worker"
+    )

lionagi/integrations/storage/storage_util.py

@@ -29,10 +29,10 @@ def output_node_list(structure):
     structure_output = {
         "id": structure.id_,
         "timestamp": structure.timestamp,
-        "type": structure.class_name(),
+        "type": structure.class_name,
     }
     summary_list.append(structure_output.copy())
-    structure_output["head_nodes"] = [i.id_ for i in structure.get_heads()]
+    structure_output["head_nodes"] = json.dumps([i.id_ for i in structure.get_heads()])
     # structure_output['nodes'] = json.dumps([i for i in structure.internal_nodes.keys()])
     # structure_output['edges'] = json.dumps([i for i in structure.internal_edges.keys()])
     output[structure_output["type"]] = [structure_output]
@@ -40,7 +40,7 @@ def output_node_list(structure):
         node_output = {
             "id": node.id_,
             "timestamp": node.timestamp,
-            "type": node.class_name(),
+            "type": node.class_name,
         }
         summary_list.append(node_output.copy())
         if isinstance(node, System) or isinstance(node, Instruction):
@@ -286,4 +286,4 @@ class ParseNode:
             if key == "self":
                 continue
             init_params[key] = args[key]
-        return cls(**init_params)
+        return cls(**init_params)

lionagi/integrations/storage/structure_excel.py

@@ -0,0 +1,268 @@
+import pandas as pd
+import json
+from pathlib import Path
+
+from lionagi.integrations.storage.storage_util import ParseNode
+from lionagi.core.execute.structure_executor import StructureExecutor
+from lionagi.core.agent.base_agent import BaseAgent
+from lionagi.core.execute.base_executor import BaseExecutor
+from lionagi.core.execute.instruction_map_executor import InstructionMapExecutor
+
+
+def excel_reload(structure_name=None, structure_id=None, dir="structure_storage"):
+    """
+    Loads a structure from an Excel file into a StructureExecutor instance.
+
+    This function uses the StructureExcel class to handle the reloading process. It identifies the
+    Excel file based on the provided structure name or ID and reloads the structure from it.
+
+    Args:
+        structure_name (str, optional): The name of the structure to reload.
+        structure_id (str, optional): The unique identifier of the structure to reload.
+        dir (str): The directory path where the Excel files are stored.
+
+    Returns:
+        StructureExecutor: An instance of StructureExecutor containing the reloaded structure.
+
+    Raises:
+        ValueError: If neither structure_name nor structure_id is provided, or if multiple or no files
+                    are found matching the criteria.
+    """
+    excel_structure = StructureExcel(structure_name, structure_id, dir)
+    excel_structure.reload()
+    return excel_structure.structure
+
+
+class StructureExcel:
+    """
+    Manages the reloading of structures from Excel files.
+
+    This class handles the identification and parsing of structure data from an Excel workbook. It supports
+    loading from specifically named Excel files based on the structure name or ID.
+
+    Attributes:
+        structure (StructureExecutor): The loaded structure, ready for execution.
+        default_agent_executable (BaseExecutor): The default executor for agents within the structure.
+
+    Methods:
+        get_heads(): Retrieves the head nodes of the loaded structure.
+        parse_node(info_dict): Parses a node from the structure based on the provided dictionary.
+        get_next(node_id): Gets the next nodes connected by outgoing edges from a given node.
+        relate(parent_node, node): Relates two nodes within the structure based on the edge definitions.
+        parse(node_list, parent_node=None): Recursively parses nodes and their relationships from the structure.
+        reload(): Reloads the structure from the Excel file based on the initially provided parameters.
+    """
+    structure: StructureExecutor = StructureExecutor()
+    default_agent_executable: BaseExecutor = InstructionMapExecutor()
+
+    def __init__(self, structure_name=None, structure_id=None, file_path="structure_storage"):
+        """
+        Initializes the StructureExcel class with specified parameters.
+
+        This method sets up the paths and reads the Excel file, preparing the internal dataframes used for
+        structure parsing.
+
+        Args:
+            structure_name (str, optional): The name of the structure to reload.
+            structure_id (str, optional): The unique identifier of the structure to reload.
+            file_path (str): The base path where the Excel files are stored.
+
+        Raises:
+            ValueError: If both structure_name and structure_id are provided but do not correspond to a valid file,
+                        or if multiple or no files are found when one of the identifiers is provided.
+        """
+        self.file_path = file_path
+        if not structure_name and not structure_id:
+            raise ValueError("Please provide the structure name or id")
+        if structure_name and structure_id:
+            self.filename = f"{file_path}/{structure_name}_{structure_id}.xlsx"
+            self.file = pd.read_excel(self.filename, sheet_name=None)
+        elif structure_name and not structure_id:
+            dir_path = Path(file_path)
+            files = list(dir_path.glob(f"{structure_name}*.xlsx"))
+            filename = []
+            for file in files:
+                try:
+                    name = file.name
+                    name = name.rsplit("_", 1)[0]
+                    if name == structure_name:
+                        filename.append(file.name)
+                except:
+                    continue
+            if len(filename) > 1:
+                raise ValueError(
+                    f"Multiple files starting with the same structure name {structure_name} has found, please specify the structure id")
+            self.filename = f"{file_path}/{filename[0]}"
+            self.file = pd.read_excel(self.filename, sheet_name=None)
+        elif structure_id and not structure_name:
+            dir_path = Path(file_path)
+            files = list(dir_path.glob(f"*{structure_id}.xlsx"))
+            filename = [file.name for file in files]
+            if len(filename) > 1:
+                raise ValueError(
+                    f"Multiple files with the same structure id {structure_id} has found, please double check the stored structure")
+            self.filename = f"{file_path}/{filename[0]}"
+            self.file = pd.read_excel(self.filename, sheet_name=None)
+        self.nodes = self.file["Nodes"]
+        self.edges = self.file["Edges"]
+
+    def get_heads(self):
+        """
+        Retrieves the list of head node identifiers from the loaded structure data.
+
+        This method parses the 'StructureExecutor' sheet in the loaded Excel file to extract the list of head nodes.
+
+        Returns:
+            list: A list of identifiers for the head nodes in the structure.
+        """
+        structure_df = self.file["StructureExecutor"]
+        head_list = json.loads(structure_df["head_nodes"].iloc[0])
+        return head_list
+
+    def _reload_info_dict(self, node_id):
+        """
+        Retrieves detailed information about a specific node from the Excel file based on its identifier.
+
+        This method looks up a node's information within the loaded Excel sheets and returns a dictionary
+        containing all the relevant details.
+
+        Args:
+            node_id (str): The identifier of the node to look up.
+
+        Returns:
+            dict: A dictionary containing the properties and values for the specified node.
+        """
+        node_type = self.nodes[self.nodes["id"] == node_id]["type"].iloc[0]
+        node_file = self.file[node_type]
+        row = node_file[node_file["id"] == node_id].iloc[0]
+        info_dict = row.to_dict()
+        return info_dict
+
+    def parse_agent(self, info_dict):
+        """
+        Parses an agent node from the structure using the agent's specific details provided in a dictionary.
+
+        This method creates an agent instance based on the information from the dictionary, which includes
+        dynamically loading the output parser code.
+
+        Args:
+            info_dict (dict): A dictionary containing details about the agent node, including its class, structure ID,
+                              and output parser code.
+
+        Returns:
+            BaseAgent: An initialized agent object.
+        """
+        output_parser = ParseNode.convert_to_def(info_dict["output_parser"])
+
+        structure_excel = StructureExcel(structure_id=info_dict["structure_id"], file_path=self.file_path)
+        structure_excel.reload()
+        structure = structure_excel.structure
+        agent = BaseAgent(structure=structure, executable=self.default_agent_executable, output_parser=output_parser)
+        agent.id_ = info_dict["id"]
+        agent.timestamp = info_dict["timestamp"]
+        return agent
+
+    def parse_node(self, info_dict):
+        """
+        Parses a node from its dictionary representation into a specific node type like System, Instruction, etc.
+
+        This method determines the type of node from the info dictionary and uses the appropriate parsing method
+        to create an instance of that node type.
+
+        Args:
+            info_dict (dict): A dictionary containing node data, including the node type and associated properties.
+
+        Returns:
+            Node: An instance of the node corresponding to the type specified in the info_dict.
+        """
+        if info_dict["type"] == "System":
+            return ParseNode.parse_system(info_dict)
+        elif info_dict["type"] == "Instruction":
+            return ParseNode.parse_instruction(info_dict)
+        elif info_dict["type"] == "Tool":
+            return ParseNode.parse_tool(info_dict)
+        elif info_dict["type"] == "ActionSelection":
+            return ParseNode.parse_actionSelection(info_dict)
+        elif info_dict["type"] == "BaseAgent":
+            return self.parse_agent(info_dict)
+
+    def get_next(self, node_id):
+        """
+        Retrieves the list of identifiers for nodes that are directly connected via outgoing edges from the specified node.
+
+        This method searches the 'Edges' DataFrame for all entries where the specified node is a head and returns
+        a list of the tail node identifiers.
+
+        Args:
+            node_id (str): The identifier of the node whose successors are to be found.
+
+        Returns:
+            list[str]: A list of identifiers for the successor nodes.
+        """
+        return self.edges[self.edges["head"] == node_id]["tail"].to_list()
+
+    def relate(self, parent_node, node):
+        """
+        Establishes a relationship between two nodes in the structure based on the Excel data for edges.
+
+        This method looks up the edge details connecting the two nodes and applies any conditions associated
+        with the edge to the structure being rebuilt.
+
+        Args:
+            parent_node (Node): The parent node in the relationship.
+            node (Node): The child node in the relationship.
+
+        Raises:
+            ValueError: If there are issues with the edge data such as multiple undefined edges.
+        """
+        if not parent_node:
+            return
+        row = self.edges[(self.edges["head"] == parent_node.id_) & (self.edges["tail"] == node.id_)]
+        if len(row) > 1:
+            raise ValueError(
+                f"currently does not support handle multiple edges between two nodes, Error node: from {parent_node.id_} to {node.id_}")
+        if row['condition'].isna().any():
+            self.structure.relate_nodes(parent_node, node)
+        else:
+            cond = json.loads(row["condition"].iloc[0])
+            cond_cls = cond["class"]
+            cond_row = self.file["EdgesCondClass"][self.file["EdgesCondClass"]["class_name"] == cond_cls]
+            cond_code = cond_row["class"].iloc[0]
+            condition = ParseNode.parse_condition(cond, cond_code)
+            self.structure.relate_nodes(parent_node, node, condition=condition)
+
+    def parse(self, node_list, parent_node=None):
+        """
+        Recursively parses a list of nodes and establishes their interconnections based on the Excel data.
+
+        This method processes each node ID in the list, parsing individual nodes and relating them according
+        to their connections defined in the Excel file.
+
+        Args:
+            node_list (list[str]): A list of node identifiers to be parsed.
+            parent_node (Node, optional): The parent node to which the nodes in the list are connected.
+
+        Raises:
+            ValueError: If an error occurs during parsing or relating nodes.
+        """
+        for node_id in node_list:
+            info_dict = self._reload_info_dict(node_id)
+            node = self.parse_node(info_dict)
+
+            if node.id_ not in self.structure.internal_nodes:
+                self.structure.add_node(node)
+            self.relate(parent_node, node)
+
+            next_node_list = self.get_next(node_id)
+            self.parse(next_node_list, node)
+
+    def reload(self):
+        """
+        Reloads the entire structure from the Excel file.
+
+        This method initializes a new StructureExecutor and uses the Excel data to rebuild the entire structure,
+        starting from the head nodes and recursively parsing and connecting all nodes defined within.
+        """
+        self.structure = StructureExecutor()
+        heads = self.get_heads()
+        self.parse(heads)

lionagi/integrations/storage/to_excel.py

@@ -4,9 +4,7 @@ from lionagi.libs import SysUtil
 from lionagi.integrations.storage.storage_util import output_node_list, output_edge_list
 
 
-def _output_excel(
-    node_list, node_dict, edge_list, edge_cls_list, filename="structure_storage"
-):
+def _output_excel(node_list, node_dict, edge_list, edge_cls_list, structure_name, dir="structure_storage"):
     """
     Writes provided node and edge data into multiple sheets of a single Excel workbook.
 
@@ -20,7 +18,7 @@ def _output_excel(
                           node attributes for nodes of that type.
         edge_list (list): A list of dictionaries where each dictionary contains attributes of a single edge.
         edge_cls_list (list): A list of dictionaries where each dictionary contains attributes of edge conditions.
-        filename (str): The base name for the output Excel file. The '.xlsx' extension will be added
+        structure_name (str): The base name for the output Excel file. The '.xlsx' extension will be added
                         automatically if not included.
 
     Returns:
@@ -31,20 +29,30 @@ def _output_excel(
     """
     SysUtil.check_import("openpyxl")
 
+    structure_id = ""
+
     tables = {"Nodes": pd.DataFrame(node_list), "Edges": pd.DataFrame(edge_list)}
     if edge_cls_list:
         tables["EdgesCondClass"] = pd.DataFrame(edge_cls_list)
     for i in node_dict:
+        if i == "StructureExecutor":
+            structure_node = node_dict[i][0]
+            structure_node["name"] = structure_name
+            structure_id = structure_node["id"]
         tables[i] = pd.DataFrame(node_dict[i])
 
-    filename = filename + ".xlsx"
+    import os
+    filepath = os.path.join(dir, f"{structure_name}_{structure_id}.xlsx")
+
+    if not os.path.exists(dir):
+        os.makedirs(dir)
 
-    with pd.ExcelWriter(filename) as writer:
+    with pd.ExcelWriter(filepath) as writer:
         for i in tables:
             tables[i].to_excel(writer, sheet_name=i, index=False)
 
 
-def to_excel(structure, filename="structure_storage"):
+def to_excel(structure, structure_name, dir="structure_storage"):
     """
     Converts a structure into a series of Excel sheets within a single workbook.
 
@@ -55,7 +63,7 @@ def to_excel(structure, filename="structure_storage"):
     Args:
         structure: An object representing the structure to be serialized. This should have methods
                    to return lists of nodes and edges suitable for output.
-        filename (str): The base name of the output Excel file. The '.xlsx' extension will be added
+        structure_name (str): The base name of the output Excel file. The '.xlsx' extension will be added
                         automatically if not included.
 
     Returns:
@@ -64,4 +72,5 @@ def to_excel(structure, filename="structure_storage"):
     node_list, node_dict = output_node_list(structure)
     edge_list, edge_cls_list = output_edge_list(structure)
 
-    _output_excel(node_list, node_dict, edge_list, edge_cls_list, filename)
+    _output_excel(node_list, node_dict, edge_list, edge_cls_list, structure_name, dir)
+

lionagi/libs/__init__.py

@@ -16,6 +16,9 @@ from lionagi.libs.ln_api import (
     PayloadPackage,
 )
 
+from lionagi.libs.ln_validate import validation_funcs
+
+
 __all__ = [
     "SysUtil",
     "convert",
@@ -31,4 +34,5 @@ __all__ = [
     "StatusTracker",
     "SimpleRateLimiter",
     "CallDecorator",
+    "validation_funcs",
 ]

lionagi/tests/test_core/generic/test_component.py

@@ -0,0 +1,89 @@
+import unittest
+from pydantic import BaseModel
+from pandas import Series
+import pandas as pd
+from lionagi.core.generic.component import BaseNode
+
+class TestBaseNode(unittest.TestCase):
+
+    def test_basic_properties(self):
+        node = BaseNode(content="Hello World")
+        self.assertIsInstance(node.id_, str)
+        self.assertEqual(len(node.id_), 32)
+        self.assertEqual(node.content, "Hello World")
+        self.assertEqual(node.metadata, {})
+
+    def test_serialization_json(self):
+        node = BaseNode(content="Hello World")
+        self.assertIn('id_', node.to_json_str())
+
+    def test_serialization_dict(self):
+        node = BaseNode(content="Hello World")
+        result = node.to_dict()
+        self.assertIn('id_', result)
+        self.assertEqual(result['content'], "Hello World")
+
+    def test_serialization_xml(self):
+        node = BaseNode(content="Hello World")
+        xml_str = node.to_xml()
+        self.assertIn('<BaseNode>', xml_str)
+        self.assertIn('</BaseNode>', xml_str)
+
+    def test_conversion_to_pandas_series(self):
+        node = BaseNode(content="Hello World")
+        series = node.to_pd_series()
+        self.assertIsInstance(series, Series)
+        self.assertEqual(series['content'], "Hello World")
+
+    def test_field_management(self):
+        node = BaseNode(content="Hello World")
+        node._add_field("new_field", annotation=str, default="default value")
+        self.assertIn('new_field', node._field_annotations)
+        self.assertEqual(getattr(node, 'new_field', None), "default value")
+
+    def test_creation_from_dict(self):
+        data = {"a": 1, "b": 2}
+        node = BaseNode.from_obj(data)
+        self.assertIn('a', node.to_dict())
+
+    def test_creation_from_json_str(self):
+        json_str = '{"a": 1, "b": 2}'
+        node = BaseNode.from_obj(json_str)
+        self.assertIn('a', node.to_dict())
+
+    def test_creation_from_fuzzy_json_str(self):
+        json_str = '{"name": "John", "age": 30, "city": ["New York", "DC", "LA"]'
+        node = BaseNode.from_obj(json_str, fuzzy_parse=True)
+        self.assertIn('name', node.to_dict())
+
+    def test_creation_from_pandas_series(self):
+        series = pd.Series({"a": 1, "b": 2})
+        node = BaseNode.from_obj(series)
+        self.assertIn('a', node.to_dict())
+
+    def test_creation_from_pandas_dataframe(self):
+        df = pd.DataFrame({"a": [1, 2], "b": [3, 4]}, index=["row1", "row2"])
+        nodes = BaseNode.from_obj(df)
+        self.assertIsInstance(nodes, list)
+        self.assertEqual(len(nodes), 2)
+
+    def test_creation_from_pydantic_model(self):
+        class CustomModel(BaseModel):
+            a: int
+            b: str
+
+        custom_model = CustomModel(a=1, b="hello")
+        node = BaseNode.from_obj(custom_model)
+        self.assertIn('a', node.to_dict())
+
+    def test_metadata_manipulation(self):
+        node = BaseNode()
+        node.meta_insert("key", "value")
+        self.assertEqual(node.metadata['key'], "value")
+        node.meta_change_key("key", "new_key")
+        self.assertIn("new_key", node.metadata)
+        node.meta_merge({"additional_key": "additional_value"})
+        self.assertIn("additional_key", node.metadata)
+
+if __name__ == '__main__':
+    unittest.main()

lionagi/version.py

@@ -1 +1 @@
-__version__ = "0.1.1"
+__version__ = "0.1.2"

{lionagi-0.1.1.dist-info → lionagi-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lionagi
-Version: 0.1.1
+Version: 0.1.2
 Summary: Towards automated general intelligence.
 Author: HaiyangLi
 Author-email: Haiyang Li <ocean@lionagi.ai>