lionagi 0.0.112__py3-none-any.whl → 0.0.113__py3-none-any.whl

lionagi/core/sessions.py

@@ -1,57 +1,138 @@
 import json
 from typing import Any
 from dotenv import load_dotenv
-load_dotenv()
 
-from .conversations import Conversation
-from ..utils.sys_utils import to_list
 from ..schema import DataLogger
-from ..service_.service_utils import StatusTracker
-from ..tools.tool_utils import ToolManager
-from ..service_.oai import OpenAIService
-from ..endpoint.chat_completion import ChatCompletion
-
-from ..llm_configs import oai_llmconfig, oai_schema
+from ..utils import lcall, alcall
+from ..services import OpenAIService, ChatCompletion
+from ..core.conversations import Conversation
+from ..objs.tool_registry import ToolManager
+from ..configs.oai_configs import oai_schema
 
-status_tracker = StatusTracker()
+load_dotenv()
 OAIService = OpenAIService()
 
 
 class Session:
+    """
+    The Session class is responsible for managing a conversation session with a given system,
+    handling the logging of data, and invoking tools as part of the conversation.
 
-    def __init__(self, system, dir=None, llmconfig=oai_llmconfig, service=OAIService):
+    Attributes:
+        conversation (Conversation): An object to manage the conversation flow and history.
+        
+        system (str): The name of the system with which the conversation is happening.
+        
+        llmconfig (dict): Configuration for the language model.
+        
+        _logger (DataLogger): An object for logging conversation data.
+        
+        service (OpenAIService): A service object for interacting with OpenAI APIs.
+        
+        tool_manager (ToolManager): An object to manage the registration and invocation of tools.
+    """
+
+    def __init__(
+        self, system, dir=None, llmconfig=oai_schema['chat']['config'], 
+        service=OAIService
+        ):
+        """
+        Initializes the Session object.
+
+        Args:
+            system (str): The name of the system with which the session is initiated.
+            
+            dir (str, optional): The directory for saving logs. Defaults to None.
+            
+            llmconfig (dict): Configuration for the language model. Defaults to chat config schema.
+            
+            service (OpenAIService): The service object for API interactions. Defaults to an instance of OpenAIService.
+        """
 
         self.conversation = Conversation()
         self.system = system
         self.llmconfig = llmconfig
-        self._logger = DataLogger(dir=dir)
+        self.logger_ = DataLogger(dir=dir)
         self.service = service
-        self._toolmanager = ToolManager()
+        self.tool_manager = ToolManager()
     
     def set_dir(self, dir):
-        self._logger.dir = dir
+        """
+        Sets the directory where data logs should be saved.
+        
+        Args:
+            dir (str): The path to the directory for saving logs.
+        """
+        self.logger_.dir = dir
     
     def set_system(self, system):
+        """
+        Changes the system associated with the conversation.
+        
+        Args:
+            system (str): The name of the new system for the conversation.
+        """
         self.conversation.change_system(system)
     
     def set_llmconfig(self, llmconfig):
+        """
+        Updates the language model configuration.
+        
+        Args:
+            llmconfig (dict): The new configuration for the language model.
+        """
         self.llmconfig = llmconfig
     
     def set_service(self, service):
+        """
+        Sets the service object used for API interactions.
+        
+        Args:
+            service (OpenAIService): The new service object.
+        """
         self.service = service
     
     async def _output(self, invoke=True, out=True):
+        """
+        Processes the output from the conversation, possibly invoking tools and returning the latest response.
+        
+        Args:
+            invoke (bool): Indicates whether to invoke tools based on the latest response. Defaults to True.
+            
+            out (bool): Determines whether to return the latest response content. Defaults to True.
+        
+        Returns:
+            The content of the latest response if out is True. Otherwise, returns None.
+        """
         if invoke:
             try: 
-                func, args = self._toolmanager._get_function_call(self.conversation.responses[-1]['content'])
-                outs = await self._toolmanager.invoke(func, args)
-                self.conversation.add_messages(response=outs)
+                # func, args = self.tool_manager._get_function_call(self.conversation.responses[-1]['content'])
+                # outs = await self.tool_manager.invoke(func, args)
+                # self.conversation.add_messages(response=outs)
+
+                tool_uses = json.loads(self.conversation.responses[-1]['content'])
+                if 'function_list' in tool_uses.keys():
+                    func_calls = lcall(tool_uses['function_list'], self.tool_manager._get_function_call)
+                else:
+                    func_calls = lcall(tool_uses['tool_uses'], self.tool_manager._get_function_call)
+
+                outs = await alcall(func_calls, self.tool_manager.invoke)
+                for out, f in zip(outs, func_calls):
+                    response = {"function": f[0], "arguments": f[1], "output": out}
+                    self.conversation.add_messages(response=response)
+
             except:
                 pass
         if out:
             return self.conversation.responses[-1]['content']
     
     def _is_invoked(self):
+        """
+        Checks if the last message in the conversation indicates a function call result.
+        
+        Returns:
+            bool: True if the last message is a function call result, False otherwise.
+        """
         msg = self.conversation.messages[-1]
         try: 
             if "function call result" in json.loads(msg['content']).keys():
@@ -60,28 +141,83 @@ class Session:
             return False    
 
     def register_tools(self, tools, update=False, new=False, prefix=None, postfix=None):
+        """
+        Registers a list of tools to the tool manager and updates the language model configuration.
+        
+        Args:
+            tools: A single tool or a list of tools to be registered.
+            update (bool): If True, update existing tools. Defaults to False.
+            new (bool): If True, add as new tools. Defaults to False.
+            prefix: A prefix added to all tool names. Defaults to None.
+            postfix: A postfix added to all tool names. Defaults to None.
+        """
         if not isinstance(tools, list):
             tools=[tools]
-        self._toolmanager.register_tools(tools=tools, update=update, new=new, prefix=prefix, postfix=postfix)
+        self.tool_manager.register_tools(tools=tools, update=update, new=new, prefix=prefix, postfix=postfix)
+        tools_schema = lcall(tools, lambda tool: tool.to_dict()['schema_'])
+        if self.llmconfig['tools'] is None:
+            self.llmconfig['tools'] = tools_schema
+        else:
+            self.llmconfig['tools'] += tools_schema
     
-    async def initiate(self, instruction, system=None, context=None, name=None, invoke=True, out=True, **kwargs) -> Any:
+    async def initiate(self, instruction, system=None, context=None, 
+                       name=None, invoke=True, out=True, **kwargs) -> Any:
+        """
+        Initiates a conversation with an instruction and possibly additional context.
+        
+        Args:
+            instruction (str): The initial instruction for the conversation.
+            system (str, optional): The name of the system to be used. If None, defaults to current system.
+            context (str, optional): Additional context for the conversation. Defaults to None.
+            name (str, optional): The name associated with the conversation. Defaults to None.
+            invoke (bool): Indicates whether to invoke tools. Defaults to True.
+            out (bool): Determines whether to return the latest response content. Defaults to True.
+            **kwargs: Additional keyword arguments for language model configuration.
+        
+        Returns:
+            The output of the conversation if out is True, otherwise None.
+        """
         config = {**self.llmconfig, **kwargs}
         system = system or self.system
         self.conversation.initiate_conversation(system=system, instruction=instruction, context=context, name=name)
-        await self._call_chatcompletion(**config)
+        await self.call_chatcompletion(**config)
         
         return await self._output(invoke, out)
 
-    async def followup(self, instruction, system=None, context=None, out=True, name=None, invoke=True, **kwargs) -> Any:
+    async def followup(self, instruction, system=None, context=None, 
+                       out=True, name=None, invoke=True, **kwargs) -> Any:
+        """
+        Continues the conversation with a follow-up instruction.
+        
+        Args:
+            instruction (str): The follow-up instruction for the conversation.
+            system (str, optional): The name of the system to be used. If None, defaults to current system.
+            context (str, optional): Additional context for the conversation. Defaults to None.
+            out (bool): Determines whether to return the latest response content. Defaults to True.
+            name (str, optional): The name associated with the conversation. Defaults to None.
+            invoke (bool): Indicates whether to invoke tools. Defaults to True.
+            **kwargs: Additional keyword arguments for language model configuration.
+        
+        Returns:
+            The output of the conversation if out is True, otherwise None.
+        """
         if system:
             self.conversation.change_system(system)
         self.conversation.add_messages(instruction=instruction, context=context, name=name)
         config = {**self.llmconfig, **kwargs}
-        await self._call_chatcompletion(**config)
+        await self.call_chatcompletion(**config)
 
         return await self._output(invoke, out)
 
     async def auto_followup(self, instruct, num=3, **kwargs):
+        """
+        Automatically generates follow-up messages based on whether the last response invoked a tool.
+        
+        Args:
+            instruct (str): The instruction to pass for follow-up.
+            num (int): The number of follow-ups to attempt. Defaults to 3.
+            **kwargs: Additional keyword arguments for the follow-up process.
+        """
         cont_ = True
         while num > 0 and cont_ is True:
             await self.followup(instruct, tool_choice="auto", **kwargs)
@@ -91,22 +227,33 @@ class Session:
             await self.followup(instruct, **kwargs)
 
     def messages_to_csv(self, dir=None, filename="messages.csv", **kwargs):
-        dir = dir or self._logger.dir
+        """
+        Exports the conversation messages to a CSV file.
+        
+        Args:
+            dir (str, optional): The directory where the CSV should be saved. Defaults to the logger's directory.
+            filename (str): The name of the CSV file. Defaults to "messages.csv".
+            **kwargs: Additional keyword arguments passed to the CSV writing function.
+        
+        Raises:
+            ValueError: If no directory is specified.
+        """
+        dir = dir or self.logger_.dir
         if dir is None:
             raise ValueError("No directory specified.")
         self.conversation.msg.to_csv(dir=dir, filename=filename, **kwargs)
         
     def log_to_csv(self, dir=None, filename="llmlog.csv", **kwargs):
-        dir = dir or self._logger.dir
+        dir = dir or self.logger_.dir
         if dir is None:
             raise ValueError("No directory specified.")
-        self._logger.to_csv(dir=dir, filename=filename, **kwargs)
+        self.logger_.to_csv(dir=dir, filename=filename, **kwargs)
     
-    async def _call_chatcompletion(self, schema=oai_schema, **kwargs):
+    async def call_chatcompletion(self, schema=oai_schema['chat'], **kwargs):
         payload = ChatCompletion.create_payload(messages=self.conversation.messages, schema=schema, llmconfig=self.llmconfig,**kwargs)
         completion = await self.service.serve(payload=payload)
         if "choices" in completion:
-            self._logger({"input":payload, "output": completion})
+            self.logger_({"input":payload, "output": completion})
             self.conversation.add_messages(response=completion['choices'][0])
             self.conversation.responses.append(self.conversation.messages[-1])
             self.conversation.response_counts += 1

lionagi/datastore/__init__.py

@@ -0,0 +1 @@
+# TODO

lionagi/loader/__init__.py

@@ -1,7 +1,12 @@
-from .load_utils import dir_to_files, dir_to_path, file_to_chunks
+from .reader import load, ReaderType, text_reader
+from .chunker import chunk, datanodes_convert, ChunkerType, text_chunker
 
 __all__ = [
-    "dir_to_files",
-    "dir_to_path",
-    "file_to_chunks"
+    'load',
+    'chunk',
+    'datanodes_convert',
+    'text_reader',
+    'text_chunker',
+    'ReaderType',
+    'ChunkerType'
 ]

lionagi/loader/chunker.py

@@ -0,0 +1,157 @@
+from enum import Enum
+from typing import Union, Callable
+
+from ..bridge.langchain import langchain_text_splitter, from_langchain
+from ..bridge.llama_index import llama_index_node_parser, from_llama_index
+from ..schema.base_schema import DataNode
+from ..utils import lcall, file_to_chunks
+
+# define an enum to represent different types of chunkers
+class ChunkerType(str, Enum):
+    PLAIN = 'plain'                 # default
+    LANGCHAIN = 'langchain'         # using langchain functions
+    LLAMAINDEX = 'llama_index'      # using llamaindex functions
+    SELFDEFINED = 'self_defined'    # create custom functions
+
+# Function to convert documents to a specific format based on the chunker type
+def datanodes_convert(documents, chunker_type):
+    """
+    Converts a lionagi DataNode documents to a specific format based on the chunker type.
+    
+    Args:
+        documents (List[DataNode]): A list of DataNode instances to be converted.
+        
+        chunker_type (ChunkerType): The chunker type to determine the conversion format.
+    
+    Returns:
+        List[DataNode]: The list of converted DataNode instances.
+    """
+    for i in range(len(documents)):
+        if type(documents[i]) == DataNode:
+            if chunker_type == ChunkerType.LLAMAINDEX:
+                documents[i] = documents[i].to_llama_index()
+            elif chunker_type == ChunkerType.LANGCHAIN:
+                documents[i] = documents[i].to_langchain()
+    return documents
+
+# Function to chunk text documents
+def text_chunker(documents, args, kwargs):
+    """
+    Chunks text documents into smaller pieces.
+    
+    Args:
+        documents (List[DataNode]): A list of DataNode instances to be chunked.
+        args (List[Any]): Positional arguments to be passed to the chunking function.
+        kwargs (dict): Keyword arguments to be passed to the chunking function.
+    
+    Returns:
+        List[DataNode]: A list of chunked DataNode instances.
+    """
+    def chunk_node(node):
+        chunks = file_to_chunks(node.to_dict(), *args, **kwargs)
+        lcall(chunks, lambda chunk: chunk.pop('node_id'))
+        chunk_nodes = lcall(chunks, lambda x: DataNode(**x))
+        return chunk_nodes
+
+    nodes = []
+    for doc in documents:
+        nodes += chunk_node(doc)
+    return nodes
+
+
+def _datanode_parser(nodes, parser):
+    """
+    Parses raw data into DataNode instances using the provided parser function.
+    
+    Args:
+        nodes (List[Any]): A list of raw data to be parsed.
+        parser (Callable): A function that parses raw data into DataNode instances.
+    
+    Returns:
+        List[DataNode]: A list of parsed DataNode instances.
+    
+    Raises:
+        ValueError: If the parser function fails.
+    """
+    try:
+        nodes = parser(nodes)
+    except Exception as e:
+        raise ValueError(f'DataNode parser {parser} failed. Error:{e}')
+    return nodes
+
+
+def chunk(documents,
+          chunker,
+          chunker_type=ChunkerType.PLAIN,
+          chunker_args=[],
+          chunker_kwargs={},
+          chunking_kwargs={},
+          documents_convert_func=None,
+          to_datanode: Union[bool, Callable] = True):
+    """
+    Chunks documents using the specified chunker and chunker type.
+    
+    Args:
+        documents (List[Any]): A list of documents to be chunked.
+        chunker (Callable): The chunking function to be used.
+        chunker_type (ChunkerType): The type of the chunker. Defaults to ChunkerType.PLAIN.
+        chunker_args (List[Any]): Positional arguments for the chunker function. Defaults to an empty list.
+        chunker_kwargs (dict): Keyword arguments for the chunker function. Defaults to an empty dict.
+        chunking_kwargs (dict): Additional keyword arguments for the chunking process. Defaults to an empty dict.
+        documents_convert_func (Callable): A function to convert documents to a specific format. Defaults to None.
+        to_datanode (Union[bool, Callable]): Determines whether to convert the result into DataNode instances, or
+                                             a callable to convert the result. Defaults to True.
+    
+    Returns:
+        List[DataNode]: A list of chunked DataNode instances after applying the chunker.
+    
+    Raises:
+        ValueError: If the chunker fails or an unsupported chunker type is provided.
+    """
+    if chunker_type == ChunkerType.PLAIN:
+        try:
+            if chunker == 'text_chunker':
+                chunker = text_chunker
+            nodes = chunker(documents, chunker_args, chunker_kwargs)
+            return nodes
+        except Exception as e:
+            raise ValueError(f'Reader {chunker} is currently not supported. Error: {e}')
+    if chunker_type == ChunkerType.LANGCHAIN:
+        if documents_convert_func:
+            documents = documents_convert_func(documents, 'langchain')
+        nodes = langchain_text_splitter(documents, chunker, chunker_args, chunker_kwargs)
+        if isinstance(to_datanode, bool) and to_datanode is True:
+            if isinstance(documents, str):
+                nodes = lcall(nodes, lambda x: DataNode(content=x))
+            else:
+                nodes = lcall(nodes, from_langchain)
+        elif isinstance(to_datanode, Callable):
+            nodes = _datanode_parser(nodes, to_datanode)
+        return nodes
+
+    elif chunker_type == ChunkerType.LLAMAINDEX:
+        if documents_convert_func:
+            documents = documents_convert_func(documents, 'llama_index')
+        nodes = llama_index_node_parser(documents, chunker, chunker_args, chunker_kwargs, chunking_kwargs)
+        if isinstance(to_datanode, bool) and to_datanode is True:
+            nodes = lcall(nodes, from_llama_index)
+        elif isinstance(to_datanode, Callable):
+            nodes = _datanode_parser(nodes, to_datanode)
+        return nodes
+
+    elif chunker_type == ChunkerType.SELFDEFINED:
+        try:
+            splitter = chunker(*chunker_args, **chunker_kwargs)
+            nodes = splitter.split(documents, **chunking_kwargs)
+        except Exception as e:
+            raise ValueError(f'Self defined chunker {chunker} is not valid. Error: {e}')
+
+        if isinstance(to_datanode, bool) and to_datanode is True:
+            raise ValueError(f'Please define a valid parser to DataNode.')
+        elif isinstance(to_datanode, Callable):
+            nodes = _datanode_parser(nodes, to_datanode)
+        return nodes
+
+    else:
+        raise ValueError(f'{chunker_type} is not supported. Please choose from {list(ChunkerType)}')
+    

lionagi/loader/reader.py

@@ -0,0 +1,124 @@
+from enum import Enum
+from typing import Union, Callable
+
+from lionagi.bridge.langchain import langchain_loader, from_langchain
+from lionagi.bridge.llama_index import llama_index_reader, from_llama_index
+from lionagi.utils.call_util import lcall
+from lionagi.utils.load_utils import dir_to_nodes
+
+
+class ReaderType(str, Enum):
+    PLAIN = 'PLAIN'
+    LANGCHAIN = 'langchain'
+    LLAMAINDEX = 'llama_index'
+    SELFDEFINED = 'self_defined'
+
+
+def _datanode_parser(nodes, parser):
+    """
+    Parses a list of nodes using the given parser function.
+    
+    Args:
+        nodes (List[Any]): The list of nodes to be parsed.
+        
+        parser (Callable): The parser function to transform nodes into DataNode instances.
+    
+    Returns:
+        List[Any]: A list of parsed nodes.
+    
+    Raises:
+        ValueError: If the parser function fails.
+    """
+    try:
+        nodes = parser(nodes)
+    except Exception as e:
+        raise ValueError(f'DataNode parser {parser} failed. Error:{e}')
+    return nodes
+
+
+def text_reader(args, kwargs):
+    """
+    Reads text files from a directory and converts them to DataNode instances.
+    
+    Args:
+        args (List[Any]): Positional arguments for the dir_to_nodes function.
+        
+        kwargs (dict): Keyword arguments for the dir_to_nodes function.
+    
+    Returns:
+        List[Any]: A list of DataNode instances.
+    """
+    return dir_to_nodes(*args, **kwargs)
+
+
+def load(reader: Union[str, Callable],
+         reader_type=ReaderType.PLAIN,
+         reader_args=[],
+         reader_kwargs={},
+         load_args=[],
+         load_kwargs={},
+         to_datanode: Union[bool, Callable] = True):
+    """
+    Loads documents using the specified reader and reader type.
+    
+    Args:
+        reader (Union[str, Callable]): The reader function or its name as a string.
+        
+        reader_type (ReaderType): The type of the reader. Defaults to ReaderType.PLAIN.
+        
+        reader_args (List[Any]): Positional arguments for the reader function. Defaults to an empty list.
+        
+        reader_kwargs (dict): Keyword arguments for the reader function. Defaults to an empty dict.
+        
+        load_args (List[Any]): Positional arguments for the loader function. Defaults to an empty list.
+        
+        load_kwargs (dict): Keyword arguments for the loader function. Defaults to an empty dict.
+        
+        to_datanode (Union[bool, Callable]): Determines whether to convert the result into DataNode instances, or
+                                             a callable to convert the result. Defaults to True.
+    
+    Returns:
+        List[Any]: A list of loaded and potentially parsed documents.
+    
+    Raises:
+        ValueError: If the reader fails or an unsupported reader type is provided.
+    """
+    if reader_type == ReaderType.PLAIN:
+        try:
+            if reader == 'text_reader':
+                reader = text_reader
+            nodes = reader(reader_args, reader_kwargs)
+            return nodes
+        except Exception as e:
+            raise ValueError(f'Reader {reader} is currently not supported. Error: {e}')
+    if reader_type == ReaderType.LANGCHAIN:
+        nodes = langchain_loader(reader, reader_args, reader_kwargs)
+        if isinstance(to_datanode, bool) and to_datanode is True:
+            nodes = lcall(nodes, from_langchain)
+        elif isinstance(to_datanode, Callable):
+            nodes = _datanode_parser(nodes, to_datanode)
+        return nodes
+
+    elif reader_type == ReaderType.LLAMAINDEX:
+        nodes = llama_index_reader(reader, reader_args, reader_kwargs, load_args, load_kwargs)
+        if isinstance(to_datanode, bool) and to_datanode is True:
+            nodes = lcall(nodes, from_llama_index)
+        elif isinstance(to_datanode, Callable):
+            nodes = _datanode_parser(nodes, to_datanode)
+        return nodes
+
+    elif reader_type == ReaderType.SELFDEFINED:
+        try:
+            loader = reader(*reader_args, **reader_kwargs)
+            nodes = loader.load(*load_args, **load_kwargs)
+        except Exception as e:
+            raise ValueError(f'Self defined reader {reader} is not valid. Error: {e}')
+
+        if isinstance(to_datanode, bool) and to_datanode is True:
+            raise ValueError(f'Please define a valid parser to DataNode.')
+        elif isinstance(to_datanode, Callable):
+            nodes = _datanode_parser(nodes, to_datanode)
+        return nodes
+
+    else:
+        raise ValueError(f'{reader_type} is not supported. Please choose from {list(ReaderType)}')

lionagi/objs/__init__.py

@@ -0,0 +1,7 @@
+# # from .messenger import Messenger
+# from .tool_registry import ToolRegistry
+
+# __all__ = [
+#     'Messenger',
+#     'ToolRegistry'
+# ]