lionagi 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

lionagi/core/{execute/branch_executor.py → engine/branch_engine.py}

@@ -1,23 +1,52 @@
 import contextlib
 from lionagi.libs import convert, AsyncUtil, ParseUtil
-from lionagi.core.generic import ActionNode, Edge
-from lionagi.core.mail.schema import BaseMail
-from lionagi.core.messages.schema import System, Instruction
+from lionagi.core.generic.edge import Edge
+from lionagi.core.action import ActionNode
+from lionagi.core.mail.mail import Mail
+from lionagi.core.message import System, Instruction
+from lionagi.core.collections import Pile, Progression
 
-from lionagi.core.branch.branch import Branch
-from lionagi.core.execute.base_executor import BaseExecutor
+from lionagi.core.session.branch import Branch
+from lionagi.core.executor.base_executor import BaseExecutor
 
 
 class BranchExecutor(Branch, BaseExecutor):
 
+    def __init__(
+        self,
+        context=None,
+        verbose=True,
+        system=None,
+        user=None,
+        messages=None,
+        progress=None,
+        tool_manager=None,
+        tools=None,
+        imodel=None,
+        **kwargs,
+    ):
+        super().__init__(
+            system=system,
+            user=user,
+            messages=messages,
+            progress=progress,
+            tool_manager=tool_manager,
+            tools=tools,
+            imodel=imodel,
+            **kwargs,
+        )
+        self.context = context
+        self.verbose = verbose
+
     async def forward(self) -> None:
         """
         Forwards the execution by processing all pending incoming mails in each branch. Depending on the category of the mail,
         it processes starts, nodes, node lists, conditions, or ends, accordingly executing different functions.
         """
-        for key in list(self.pending_ins.keys()):
-            while self.pending_ins[key]:
-                mail = self.pending_ins[key].popleft()
+        for key in list(self.mailbox.pending_ins.keys()):
+            while self.mailbox.pending_ins[key].size() > 0:
+                mail_id = self.mailbox.pending_ins[key].popleft()
+                mail = self.mailbox.pile.pop(mail_id)
                 if mail.category == "start":
                     self._process_start(mail)
                 elif mail.category == "node":
@@ -25,9 +54,11 @@ class BranchExecutor(Branch, BaseExecutor):
                 elif mail.category == "node_list":
                     self._process_node_list(mail)
                 elif mail.category == "condition":
-                    self._process_condition(mail)
+                    await self._process_condition(mail)
                 elif mail.category == "end":
                     self._process_end(mail)
+            if self.mailbox.pending_ins[key].size() == 0:
+                self.mailbox.pending_ins.pop(key)
 
     async def execute(self, refresh_time=1) -> None:
         """
@@ -40,60 +71,57 @@ class BranchExecutor(Branch, BaseExecutor):
             await self.forward()
             await AsyncUtil.sleep(refresh_time)
 
-    async def _process_node(self, mail: BaseMail):
+    async def _process_node(self, mail: Mail):
         """
         Processes a single node based on the node type specified in the mail's package. It handles different types of nodes such as System,
         Instruction, ActionNode, and generic nodes through separate processes.
 
         Args:
-            mail (BaseMail): The mail containing the node to be processed along with associated details.
+            mail (Mail): The mail containing the node to be processed along with associated details.
 
         Raises:
             ValueError: If an invalid mail is encountered or the process encounters errors.
         """
-        if isinstance(mail.package["package"], System):
-            self._system_process(mail.package["package"], verbose=self.verbose)
+        node = mail.package.package
+        if isinstance(node, System):
+            self._system_process(node, verbose=self.verbose)
             self.send(
-                mail.sender_id,
-                "node_id",
-                {"request_source": self.id_, "package": mail.package["package"].id_},
+                recipient=mail.sender,
+                category="node_id",
+                package=node.ln_id,
+                request_source=self.ln_id,
             )
 
-        elif isinstance(mail.package["package"], Instruction):
-            await self._instruction_process(
-                mail.package["package"], verbose=self.verbose
-            )
+        elif isinstance(node, Instruction):
+            await self._instruction_process(node, verbose=self.verbose)
             self.send(
-                mail.sender_id,
-                "node_id",
-                {"request_source": self.id_, "package": mail.package["package"].id_},
+                recipient=mail.sender,
+                category="node_id",
+                package=node.ln_id,
+                request_source=self.ln_id,
             )
 
-        elif isinstance(mail.package["package"], ActionNode):
-            await self._action_process(mail.package["package"], verbose=self.verbose)
+        elif isinstance(node, ActionNode):
+            await self._action_process(node, verbose=self.verbose)
             self.send(
-                mail.sender_id,
-                "node_id",
-                {
-                    "request_source": self.id_,
-                    "package": mail.package["package"].instruction.id_,
-                },
+                recipient=mail.sender,
+                category="node_id",
+                package=node.instruction.ln_id,
+                request_source=self.ln_id,
             )
         else:
             try:
-                await self._agent_process(mail.package["package"], verbose=self.verbose)
+                await self._agent_process(node, verbose=self.verbose)
                 self.send(
-                    mail.sender_id,
-                    "node_id",
-                    {
-                        "request_source": self.id_,
-                        "package": mail.package["package"].id_,
-                    },
+                    recipient=mail.sender,
+                    category="node_id",
+                    package=node.ln_id,
+                    request_source=self.ln_id,
                 )
             except:
                 raise ValueError(f"Invalid mail to process. Mail:{mail}")
 
-    def _process_node_list(self, mail: BaseMail):
+    def _process_node_list(self, mail: Mail):
         """
         Processes a list of nodes provided in the mail, but currently only sends an end signal as multiple path selection is not supported.
 
@@ -103,28 +131,29 @@ class BranchExecutor(Branch, BaseExecutor):
         Raises:
             ValueError: When trying to process multiple paths which is currently unsupported.
         """
-        self.send(mail.sender_id, "end", {"request_source": self.id_, "package": "end"})
+        self.send(mail.sender, category="end", package="end", request_source=self.ln_id)
         self.execute_stop = True
-        raise ValueError("Multiple path selection is currently not supported")
+        raise ValueError("Multiple path selection is not supported in BranchExecutor")
 
-    def _process_condition(self, mail: BaseMail):
+    async def _process_condition(self, mail: Mail):
         """
         Processes a condition associated with an edge based on the mail's package, setting up the result of the condition check.
 
         Args:
             mail (BaseMail): The mail containing the condition to be processed.
         """
-        relationship: Edge = mail.package["package"]
-        check_result = relationship.condition(self)
+        edge: Edge = mail.package.package
+        check_result = await edge.check_condition(self)
         back_mail = {
-            "from": self.id_,
-            "edge_id": mail.package["package"].id_,
+            "from": self.ln_id,
+            "edge_id": edge.ln_id,
             "check_result": check_result,
         }
         self.send(
-            mail.sender_id,
-            "condition",
-            {"request_source": self.id_, "package": back_mail},
+            recipient=mail.sender,
+            category="condition",
+            package=back_mail,
+            request_source=self.ln_id,
         )
 
     def _system_process(self, system: System, verbose=True, context_verbose=False):
@@ -177,21 +206,22 @@ class BranchExecutor(Branch, BaseExecutor):
             )
 
         if self.context:
-            instruction.content.update({"context": self.context})
-            self.context_log.append(self.context)
+            result = await self.chat(
+                instruction=instruction.instruct, context=self.context, **kwargs
+            )
             self.context = None
+        else:
+            result = await self.chat(instruction=instruction.instruct, **kwargs)
+            # instruction._add_context(context=self.context)
+            # self.context_log.append(self.context)
+            # self.context = None
 
-        result = await self.chat(instruction, **kwargs)
         with contextlib.suppress(Exception):
             result = ParseUtil.fuzzy_parse_json(result)
-            if "response" in result.keys():
-                result = result["response"]
-        if verbose and len(self.assistant_responses) != 0:
-            display(
-                Markdown(
-                    f"{self.last_assistant_response.sender}: {convert.to_str(result)}"
-                )
-            )
+            if "assistant_response" in result.keys():
+                result = result["assistant_response"]
+        if verbose:
+            display(Markdown(f"assistant {self.ln_id}: {convert.to_str(result)}"))
             print("-----------------------------------------------------")
 
         self.execution_responses.append(result)
@@ -209,10 +239,10 @@ class BranchExecutor(Branch, BaseExecutor):
         SysUtil.check_import("IPython")
         from IPython.display import Markdown, display
 
-        try:
-            func = getattr(self, action.action)
-        except:
-            raise ValueError(f"{action.action} is not a valid action")
+        # try:
+        #     func = getattr(self, action.action)
+        # except:
+        #     raise ValueError(f"{action.action} is not a valid action")
 
         if verbose:
             display(
@@ -221,27 +251,33 @@ class BranchExecutor(Branch, BaseExecutor):
                 )
             )
 
-        if action.tools:
-            self.register_tools(action.tools)
-        if self.context:
-            result = await func(
-                action.instruction.content["instruction"],
-                context=self.context,
-                tools=action.tools,
-                **action.action_kwargs,
-            )
-            self.context = None
-        else:
-            result = await func(
-                action.instruction.content, tools=action.tools, **action.action_kwargs
-            )
+        # if action.tools:
+        #     self.register_tools(action.tools)
+        # if self.context:
+        # result = await self.direct(
+        #     action.directive,
+        #     instruction=action.instruction.instruct,
+        #     context=self.context,
+        #     tools=action.tools,
+        #     **action.directive_kwargs,
+        # )
+        result = await action.invoke(branch=self, context=self.context)
+        self.context = None
+        # else:
+        #     result = await self.direct(
+        #         action.directive,
+        #         instruction=action.instruction.content,
+        #         tools=action.tools,
+        #         **action.directive_kwargs
+        #     )
 
-        if verbose and len(self.assistant_responses) != 0:
-            display(
-                Markdown(
-                    f"{self.last_assistant_response.sender}: {convert.to_str(result)}"
-                )
-            )
+        if verbose:
+            if action.directive == "chat":
+                display(Markdown(f"assistant {self.ln_id}: {convert.to_str(result)}"))
+            else:
+                display(Markdown(f"assistant {self.ln_id}:\n"))
+                for k, v in result.work_fields.items():
+                    display(Markdown(f"{k}: \n{v}\n"))
             print("-----------------------------------------------------")
 
         self.execution_responses.append(result)
@@ -254,7 +290,7 @@ class BranchExecutor(Branch, BaseExecutor):
             agent: The agent to process.
             verbose (bool): A flag indicating whether to provide verbose output (default: True).
         """
-        context = list(self.messages["content"])
+        context = [msg["content"] for msg in self.to_chat_messages()]
         if verbose:
             print("*****************************************************")
         result = await agent.execute(context)
@@ -262,12 +298,7 @@ class BranchExecutor(Branch, BaseExecutor):
         if verbose:
             print("*****************************************************")
 
-        from pandas import DataFrame
-
-        if isinstance(result, DataFrame):
-            self.context = list(result["content"])
-        else:
-            self.context = result
+        self.context = result
         self.execution_responses.append(result)
 
     def _process_start(self, mail):
@@ -277,15 +308,16 @@ class BranchExecutor(Branch, BaseExecutor):
         Args:
             mail (BaseMail): The start mail to process.
         """
-        start_mail_content = mail.package
+        start_mail_content = mail.package.package
         self.context = start_mail_content["context"]
         self.send(
-            start_mail_content["structure_id"],
-            "start",
-            {"request_source": self.id_, "package": "start"},
+            recipient=start_mail_content["structure_id"],
+            category="start",
+            package="start",
+            request_source=self.ln_id,
         )
 
-    def _process_end(self, mail: BaseMail):
+    def _process_end(self, mail: Mail):
         """
         Processes an end mail.
 
@@ -293,4 +325,9 @@ class BranchExecutor(Branch, BaseExecutor):
             mail (BaseMail): The end mail to process.
         """
         self.execute_stop = True
-        self.send(mail.sender_id, "end", {"request_source": self.id_, "package": "end"})
+        self.send(
+            recipient=mail.sender,
+            category="end",
+            package="end",
+            request_source=self.ln_id,
+        )

lionagi/core/{execute/instruction_map_executor.py → engine/instruction_map_engine.py}

@@ -1,33 +1,33 @@
 import asyncio
 from pydantic import Field
 
-from lionagi.libs import AsyncUtil
-
-from lionagi.core.mail.schema import BaseMail, MailTransfer
+from lionagi.core.mail.mail import Mail, Package
+from lionagi.core.collections import Exchange
 from lionagi.core.mail.mail_manager import MailManager
-from lionagi.core.execute.base_executor import BaseExecutor
-from lionagi.core.execute.branch_executor import BranchExecutor
+from lionagi.core.executor.base_executor import BaseExecutor
+from lionagi.core.engine.branch_engine import BranchExecutor
+from lionagi.core.collections import progression, pile, Pile
 
 
-class InstructionMapExecutor(BaseExecutor):
+class InstructionMapEngine(BaseExecutor):
     """
     Manages the execution of a mapped set of instructions across multiple branches within an executable structure.
 
     Attributes:
         branches (dict[str, BranchExecutor]): A dictionary of branch executors managing individual instruction flows.
         structure_id (str): The identifier for the structure within which these branches operate.
-        mail_transfer (MailTransfer): Handles the transfer of mail between branches and other components.
+        mail_transfer (Exchange): Handles the transfer of mail between branches and other components.
         branch_kwargs (dict): Keyword arguments used for initializing branches.
         num_end_branches (int): Tracks the number of branches that have completed execution.
         mail_manager (MailManager): Manages the distribution and collection of mails across branches.
     """
 
-    branches: dict[str, BranchExecutor] = Field(
+    branches: Pile[BranchExecutor] = Field(
         default_factory=dict, description="The branches of the instruction mapping."
     )
     structure_id: str = Field("", description="The ID of the executable structure.")
-    mail_transfer: MailTransfer = Field(
-        default_factory=MailTransfer, description="The mail transfer."
+    mail_transfer: Exchange = Field(
+        default_factory=Exchange, description="The mail transfer."
     )
     branch_kwargs: dict = Field(
         default_factory=dict,
@@ -53,9 +53,10 @@ class InstructionMapExecutor(BaseExecutor):
         Processes incoming mails, directing them appropriately based on their categories, and handles the initial setup
         of branches or the routing of node and condition mails.
         """
-        for key in list(self.pending_ins.keys()):
-            while self.pending_ins[key]:
-                mail: BaseMail = self.pending_ins[key].popleft()
+        for key in list(self.mailbox.pending_ins.keys()):
+            while self.mailbox.pending_ins[key].size() > 0:
+                mail_id = self.mailbox.pending_ins[key].popleft()
+                mail = self.mailbox.pile.pop(mail_id)
                 if mail.category == "start":
                     self._process_start(mail)
                 elif mail.category == "node_list":
@@ -65,9 +66,9 @@ class InstructionMapExecutor(BaseExecutor):
                     or (mail.category == "condition")
                     or (mail.category == "end")
                 ):
-                    mail.sender_id = self.mail_transfer.id_
-                    mail.recipient_id = mail.package["request_source"]
-                    self.mail_transfer.pending_outs.append(mail)
+                    mail.sender = self.mail_transfer.ln_id
+                    mail.recipient = mail.package.request_source
+                    self.mail_transfer.include(mail, "out")
 
     def transfer_outs(self):
         """
@@ -75,23 +76,24 @@ class InstructionMapExecutor(BaseExecutor):
         other mails to appropriate recipients.
         """
         for key in list(self.mail_transfer.pending_ins.keys()):
-            while self.mail_transfer.pending_ins[key]:
-                mail: BaseMail = self.mail_transfer.pending_ins[key].popleft()
+            while self.mail_transfer.pending_ins[key].size() > 0:
+                mail_id = self.mail_transfer.pending_ins[key].popleft()
+                mail = self.mail_transfer.pile.pop(mail_id)
                 if mail.category == "end":
                     self.num_end_branches += 1
                     if self.num_end_branches == len(
                         self.branches
                     ):  # tell when structure should stop
-                        mail.sender_id = self.id_
-                        mail.recipient_id = self.structure_id
-                        self.pending_outs.append(mail)
+                        mail.sender = self.ln_id
+                        mail.recipient = self.structure_id
+                        self.mailbox.include(mail, "out")
                         self.execute_stop = True
                 else:
-                    mail.sender_id = self.id_
-                    mail.recipient_id = self.structure_id
-                    self.pending_outs.append(mail)
+                    mail.sender = self.ln_id
+                    mail.recipient = self.structure_id
+                    self.mailbox.include(mail, "out")
 
-    def _process_start(self, start_mail: BaseMail):
+    def _process_start(self, start_mail: Mail):
         """
         Processes a start mail to initialize a new branch executor and configures it based on the mail's package content.
 
@@ -99,19 +101,20 @@ class InstructionMapExecutor(BaseExecutor):
             start_mail (BaseMail): The mail initiating the start of a new branch execution.
         """
         branch = BranchExecutor(verbose=self.verbose, **self.branch_kwargs)
-        branch.context = start_mail.package["context"]
-        self.branches[branch.id_] = branch
+        branch.context = start_mail.package.package["context"]
+        self.branches[branch.ln_id] = branch
         self.mail_manager.add_sources([branch])
-        self.structure_id = start_mail.package["structure_id"]
-        mail = BaseMail(
-            sender_id=self.id_,
-            recipient_id=self.structure_id,
-            category="start",
-            package={"request_source": branch.id_, "package": "start"},
+        self.structure_id = start_mail.package.package["structure_id"]
+
+        pack = Package(category="start", package="start", request_source=branch.ln_id)
+        mail = Mail(
+            sender=self.ln_id,
+            recipient=self.structure_id,
+            package=pack,
         )
-        self.pending_outs.append(mail)
+        self.mailbox.include(mail, "out")
 
-    def _process_node_list(self, nl_mail: BaseMail):
+    def _process_node_list(self, nl_mail: Mail):
         """
         Processes a node list mail, setting up new branches or propagating the execution context based on the node list
         provided in the mail.
@@ -119,39 +122,59 @@ class InstructionMapExecutor(BaseExecutor):
         Args:
             nl_mail (BaseMail): The mail containing a list of nodes to be processed in subsequent branches.
         """
-        source_branch_id = nl_mail.package["request_source"]
-        node_list = nl_mail.package["package"]
+        source_branch_id = nl_mail.package.request_source
+        node_list = nl_mail.package.package
         shared_context = self.branches[source_branch_id].context
         shared_context_log = self.branches[source_branch_id].context_log
         base_branch = self.branches[source_branch_id]
 
-        first_node_mail = BaseMail(
-            sender_id=self.mail_transfer.id_,
-            recipient_id=source_branch_id,
-            category="node",
-            package={"request_source": source_branch_id, "package": node_list[0]},
+        pack = Package(
+            category="node", package=node_list[0], request_source=source_branch_id
         )
-        self.mail_transfer.pending_outs.append(first_node_mail)
+        mail = Mail(
+            sender=self.mail_transfer.ln_id,
+            recipient=source_branch_id,
+            package=pack,
+        )
+        self.mail_transfer.include(mail, "out")
 
         for i in range(1, len(node_list)):
+            system = base_branch.system.clone() if base_branch.system else None
+            if system:
+                system.sender = base_branch.ln_id
+            progress = progression()
+            messages = pile()
+
+            for id_ in base_branch.progress:
+                clone_message = base_branch.messages[id_].clone()
+                progress.append(clone_message.ln_id)
+                messages.append(clone_message)
+
             branch = BranchExecutor(
                 verbose=self.verbose,
-                messages=base_branch.messages.copy(),
-                service=base_branch.service,
-                llmconfig=base_branch.llmconfig,
-                datalogger=base_branch.datalogger,
+                messages=messages,
+                user=base_branch.user,
+                system=base_branch.system.clone(),
+                progress=progress,
+                imodel=base_branch.imodel,
             )
+            for message in branch.messages:
+                message.sender = base_branch.ln_id
+                message.recipient = branch.ln_id
+
             branch.context = shared_context
             branch.context_log = shared_context_log
-            self.branches[branch.id_] = branch
+            self.branches[branch.ln_id] = branch
             self.mail_manager.add_sources([branch])
-            node_mail = BaseMail(
-                sender_id=self.mail_transfer.id_,
-                recipient_id=branch.id_,
-                category="node",
-                package={"request_source": source_branch_id, "package": node_list[i]},
+            node_pacakge = Package(
+                category="node", package=node_list[i], request_source=source_branch_id
+            )
+            node_mail = Mail(
+                sender=self.mail_transfer.ln_id,
+                recipient=branch.ln_id,
+                package=node_pacakge,
             )
-            self.mail_transfer.pending_outs.append(node_mail)
+            self.mail_transfer.include(node_mail, "out")
 
     async def forward(self):
         """
@@ -162,7 +185,9 @@ class InstructionMapExecutor(BaseExecutor):
         self.mail_manager.collect_all()
         self.mail_manager.send_all()
         tasks = [
-            branch.forward() for branch in self.branches.values() if branch.pending_ins
+            branch.forward()
+            for branch in self.branches.values()
+            if branch.mailbox.pending_ins
         ]
         await asyncio.gather(*tasks)
         return

lionagi/{experimental/directive/evaluator → core/engine}/script_engine.py

@@ -1,7 +1,23 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import ast
 from functools import lru_cache
 from lionagi.libs import AsyncUtil
-from .base_evaluator import BaseEvaluator
+from ..evaluator.base_evaluator import BaseEvaluator
 from .sandbox_ import SandboxTransformer