lionagi 0.0.111__py3-none-any.whl → 0.0.113__py3-none-any.whl

lionagi/__init__.py

@@ -16,9 +16,14 @@ Copyright 2023 HaiyangLi <ocean@lionagi.ai>
 
 import logging
 from .version import __version__
-from .session import *
+
 from .utils import *
-from .api import *
+from .schema import *
+from .structure import *
+from .core import *
+from .objs import *
+# from .datastore import *
+# from .structure import *
 
 
 logger = logging.getLogger(__name__)

lionagi/bridge/__init__.py

@@ -0,0 +1,7 @@
+from .langchain import from_langchain
+from .llama_index import from_llama_index
+
+__all__ = [
+    "from_langchain",
+    "from_llama_index"
+]

lionagi/bridge/langchain.py

@@ -0,0 +1,131 @@
+from typing import Union, Callable, List, Dict, Any
+from ..schema.base_schema import T, DataNode
+from ..utils.sys_util import change_dict_key
+
+
+def from_langchain(lc_doc: Any) -> T:
+    """
+    Converts a langchain document into a DataNode object.
+
+    Parameters:
+        lc_doc (Any): The langchain document to be converted.
+
+    Returns:
+        DataNode: A DataNode object created from the langchain document.
+    """
+    info_json = lc_doc.to_json()
+    info_node = {'lc_id': info_json['id']}
+    info_node = {**info_node, **info_json['kwargs']}
+    return DataNode(**info_node)
+
+def to_langchain_document(datanode: T, **kwargs: Any) -> Any:
+    """
+    Converts a DataNode into a langchain Document.
+
+    Parameters:
+        datanode (DataNode): The DataNode to be converted.
+        
+        **kwargs: Additional keyword arguments to be included in the Document.
+
+    Returns:
+        Any: A langchain Document created from the DataNode.
+    """
+    from langchain.schema import Document
+
+    dnode = datanode.to_dict()
+    change_dict_key(dnode, old_key='content', new_key='page_content')
+    change_dict_key(dnode, old_key='lc_id', new_key='id_')
+    dnode = {**dnode, **kwargs}
+    return Document(**dnode)
+
+def langchain_loader(loader: Union[str, Callable], 
+                     loader_args: List[Any] = [], 
+                     loader_kwargs: Dict[str, Any] = {}) -> Any:
+    """
+    Loads data using a specified langchain loader.
+
+    Parameters:
+        loader (Union[str, Callable]): The name of the loader function or the loader function itself.
+        
+        loader_args (List[Any]): Positional arguments to pass to the loader function.
+        
+        loader_kwargs (Dict[str, Any]): Keyword arguments to pass to the loader function.
+
+    Returns:
+        Any: The data loaded by the loader function.
+
+    Raises:
+        ValueError: If the specified loader is invalid or if the loader fails to load data.
+    """
+    import langchain.document_loaders as document_loaders
+
+    try:
+        if isinstance(loader, str):
+            loader = getattr(document_loaders, loader)
+        else:
+            loader = loader
+    except Exception as e:
+        raise ValueError(f'Invalid loader: {loader}. Error: {e}')
+
+    try:
+        loader_obj = loader(*loader_args, **loader_kwargs)
+        data = loader_obj.load()
+        return data
+    except Exception as e:
+        raise ValueError(f'Failed to load. Error: {e}')
+
+def langchain_text_splitter(data: Union[str, List],
+                            splitter: Union[str, Callable], 
+                            splitter_args: List[Any] = [], 
+                            splitter_kwargs: Dict[str, Any] = {}) -> List[str]:
+
+    import langchain.text_splitter as text_splitter
+
+    try:
+        if isinstance(splitter, str):
+            splitter = getattr(text_splitter, splitter)
+        else:
+            splitter = splitter
+    except Exception as e:
+        raise ValueError(f'Invalid text splitter: {splitter}. Error: {e}')
+
+    try:
+        splitter_obj = splitter(*splitter_args, **splitter_kwargs)
+        if isinstance(data, str):
+            chunk = splitter_obj.split_text(data)
+        else:
+            chunk = splitter_obj.split_documents(data)
+        return chunk
+    except Exception as e:
+        raise ValueError(f'Failed to split. Error: {e}')
+
+# def langchain_code_splitter(doc: str,
+#                             language: str,
+#                             splitter_args: List[Any] = [],
+#                             splitter_kwargs: Dict[str, Any] = {}) -> List[Any]:
+#     """
+#     Splits code into smaller chunks using a RecursiveCharacterTextSplitter specific to a language.
+#
+#     Parameters:
+#         doc (str): The code document to be split.
+#         language (str): The programming language of the code.
+#         splitter_args (List[Any]): Positional arguments to pass to the splitter.
+#         splitter_kwargs (Dict[str, Any]): Keyword arguments to pass to the splitter.
+#
+#     Returns:
+#         List[Any]: A list of Documents, each representing a chunk of the original code.
+#
+#     Raises:
+#         ValueError: If the splitter fails to split the code document.
+#     """
+#     from langchain.text_splitter import RecursiveCharacterTextSplitter
+#
+#     try:
+#         splitter = RecursiveCharacterTextSplitter.from_language(
+#             language=language, *splitter_args, **splitter_kwargs
+#             )
+#         docs = splitter.create_documents([doc])
+#         return docs
+#     except Exception as e:
+#         raise ValueError(f'Failed to split. Error: {e}')
+#

lionagi/bridge/llama_index.py

@@ -0,0 +1,157 @@
+from typing import Union, Callable, List, Any, Dict
+from ..schema.base_schema import DataNode, T
+from ..utils.sys_util import change_dict_key
+
+
+def from_llama_index(llama_node: Any, **kwargs: Any) -> T:
+    """
+    Converts a Llama Index node into a DataNode object.
+
+    Parameters:
+        llama_node (Any): The Llama Index node to be converted.
+        
+        **kwargs: Additional keyword arguments for JSON serialization.
+
+    Returns:
+        DataNode: A DataNode object created from the Llama Index node.
+    """
+    llama_dict = llama_node.to_dict(**kwargs)
+    return DataNode.from_dict(llama_dict)
+
+def to_llama_index_textnode(datanode: T, **kwargs: Any) -> Any:
+    """
+    Converts a DataNode into a Llama Index TextNode.
+
+    Parameters:
+        datanode (DataNode): The DataNode to be converted.
+        
+        **kwargs: Additional keyword arguments to be included in the TextNode.
+
+    Returns:
+        TextNode: A Llama Index TextNode created from the DataNode.
+    """
+    # to llama_index textnode
+    from llama_index.schema import TextNode
+
+    dnode = datanode.to_dict()
+    change_dict_key(dnode, old_key='content', new_key='text')
+    change_dict_key(dnode, old_key='node_id', new_key='id_')
+    
+    dnode = {**dnode, **kwargs}
+    return TextNode.from_dict(dnode)
+
+def get_llama_reader(reader: Union[str, Callable]) -> Callable:
+    """
+    Gets a Llama Index reader function.
+
+    Parameters:
+        reader (Union[str, Callable]): The name of the reader function or the reader function itself.
+
+    Returns:
+        Callable: The Llama Index reader function.
+
+    Raises:
+        ValueError: If the specified reader is invalid.
+    """
+    try:
+        if isinstance(reader, str):
+            if reader == 'SimpleDirectoryReader':
+                from llama_index import SimpleDirectoryReader
+                return SimpleDirectoryReader
+            else:
+                from llama_index import download_loader
+                return download_loader(reader)
+        else:
+            return reader
+    except Exception as e:
+        raise ValueError(f'Invalid reader: {reader}, Error: {e}')
+
+def llama_index_reader(reader: Union[str, Callable], 
+                       reader_args: List[Any] = [], 
+                       reader_kwargs: Dict[str, Any] = {}, 
+                       load_data_args: List[Any] = [], 
+                       load_data_kwargs: Dict[str, Any] = {}) -> List[Any]:
+    """
+    Loads documents using a specified Llama Index reader.
+
+    Parameters:
+        reader (Union[str, Callable]): The name of the reader function or the reader function itself.
+        
+        reader_args (List[Any]): Positional arguments to pass to the reader function.
+        
+        reader_kwargs (Dict[str, Any]): Keyword arguments to pass to the reader function.
+        
+        load_data_args (List[Any]): Positional arguments for the load_data method.
+        
+        load_data_kwargs (Dict[str, Any]): Keyword arguments for the load_data method.
+
+    Returns:
+        List[Any]: A list of documents loaded by the reader.
+
+    Raises:
+        ValueError: If the specified reader is invalid or if the reader fails to load documents.
+    """
+    reader = get_llama_reader(reader)
+
+    try:
+        loader = reader(*reader_args, **reader_kwargs)
+        documents = loader.load_data(*load_data_args, **load_data_kwargs)
+        return documents
+    
+    except Exception as e:
+        raise ValueError(f'Failed to read. Error: {e}')
+
+def get_llama_parser(parser: Union[str, Callable]) -> Callable:
+    import llama_index.node_parser as node_parser
+    import llama_index.text_splitter as text_splitter
+
+    try:
+        return getattr(node_parser, parser)
+    except Exception as e1:
+        try:
+            if isinstance(parser, str):
+                return getattr(text_splitter, parser)
+            else:
+                return parser
+        except Exception as e2:
+            raise ValueError(f'Invalid node parser: {parser}. Error: {e1}, {e2}')
+
+
+def llama_index_node_parser(documents: List[Any], 
+                            parser: Union[str, Callable], 
+                            parser_args: List[Any] = [], 
+                            parser_kwargs: Dict[str, Any] = {},
+                            parsing_kwargs: Dict[str, Any] = {}) -> List[Any]:
+    """
+    Parses documents into nodes using a specified Llama Index node parser.
+
+    Parameters:
+        documents (List[Any]): The documents to parse.
+        
+        parser (Union[str, Callable]): The name of the parser function or the parser function itself.
+        
+        parser_args (List[Any]): Positional arguments to pass to the parser function.
+        
+        parser_kwargs (Dict[str, Any]): Keyword arguments to pass to the parser function.
+
+    Returns:
+        List[Any]: A list of nodes parsed from the documents.
+
+    Raises:
+        ValueError: If the specified parser is invalid or if the parser fails to parse the documents.
+    """
+    parser = get_llama_parser(parser)
+
+    try:
+        parser_obj = parser(*parser_args, **parser_kwargs)
+        nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
+        return nodes
+
+    except Exception as e1:
+        try:
+            parser_obj = parser.from_defaults(*parser_args, **parser_kwargs)
+            nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
+            return nodes
+        except Exception as e2:
+            raise ValueError(f'Failed to parse. Error: {e1}, {e2}')
+        

lionagi/configs/__init__.py

@@ -0,0 +1,7 @@
+from .oai_configs import oai_schema
+from .openrouter_config import openrouter_schema
+
+__all__ = [
+    "oai_schema",
+    "openrouter_schema"
+]

lionagi/configs/oai_configs.py

@@ -0,0 +1,49 @@
+oai_chat_llmconfig = {
+    "model": "gpt-4-1106-preview",
+    "frequency_penalty": 0,
+    "max_tokens": None,
+    "n": 1,
+    "presence_penalty": 0, 
+    "response_format": {"type": "text"}, 
+    "seed": None, 
+    "stop": None,
+    "stream": False,
+    "temperature": 0.7, 
+    "top_p": 1, 
+    "tools": None,
+    "tool_choice": "none", 
+    "user": None
+    }
+
+oai_chat_schema = {
+    "required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
+    "optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"],
+    "input": "messages",
+    "config": oai_chat_llmconfig
+    }
+
+oai_finetune_llmconfig = {
+    "model": "gpt-3.5-turbo",
+    "hyperparameters": {
+        "batch_size": "auto", 
+        "learning_rate_multiplier": "auto",
+        "n_epochs": "auto"
+    },
+    "suffix": None,
+    "training_file": None,
+    }
+
+oai_finetune_schema = {
+    "required" : ["model", "training_file"],
+    "optional": ["hyperparameters", "suffix", "validate_file"],
+    "input": ["training_file"],
+    "config": oai_finetune_llmconfig 
+}
+
+
+oai_schema = {
+    
+    "chat": oai_chat_schema,
+    "finetune": oai_finetune_schema
+    
+}

lionagi/configs/openrouter_config.py

@@ -0,0 +1,49 @@
+openrouter_chat_llmconfig = {
+    "model": "gpt-4-1106-preview",
+    "frequency_penalty": 0,
+    "max_tokens": None,
+    "n": 1,
+    "presence_penalty": 0, 
+    "response_format": {"type": "text"}, 
+    "seed": None, 
+    "stop": None,
+    "stream": False,
+    "temperature": 0.7, 
+    "top_p": 1, 
+    "tools": None,
+    "tool_choice": "none", 
+    "user": None
+    }
+
+openrouter_chat_schema = {
+    "required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
+    "optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"],
+    "input": "messages",
+    "config": openrouter_chat_llmconfig
+    }
+
+openrouter_finetune_llmconfig = {
+    "model": "gpt-3.5-turbo",
+    "hyperparameters": {
+        "batch_size": "auto", 
+        "learning_rate_multiplier": "auto",
+        "n_epochs": "auto"
+    },
+    "suffix": None,
+    "training_file": None,
+    }
+
+openrouter_finetune_schema = {
+    "required" : ["model", "training_file"],
+    "optional": ["hyperparameters", "suffix", "validate_file"],
+    "input": ["training_file"],
+    "config": openrouter_finetune_llmconfig
+}
+
+
+openrouter_schema = {
+
+    "chat": openrouter_chat_schema,
+    "finetune": openrouter_finetune_schema
+    
+}

lionagi/core/__init__.py

@@ -0,0 +1,15 @@
+# from .messages import Response, Instruction, System
+from .conversations import Conversation
+from .sessions import Session
+
+# from .instruction_sets import InstructionSet
+# from .flows.flow import Flow
+
+
+__all__ = [
+    # "Response",
+    # "Instruction",
+    # "System",
+    "Conversation",
+    "Session",              #"Flow", "InstructionSet"
+]

lionagi/{session/conversation.py → core/conversations.py}

@@ -1,4 +1,4 @@
-from .message import Message
+from .messages import Message
 
 
 class Conversation:
@@ -9,22 +9,21 @@ class Conversation:
     user instructions, and assistant responses.
 
     Attributes:
-        response_counts (int):
-            The count of assistant responses in the conversation.
-        messages (list):
-            A list to store messages in the conversation.
-        msg (Message):
-            An instance of the Message class for creating messages.
-        responses (list):
-            A list to store assistant responses in the conversation.
+        response_counts (int): The count of assistant responses in the conversation.
+        messages (list): A list to store messages in the conversation.
+        msg (Message): An instance of the Message class for creating messages.
+        responses (list): A list to store assistant responses in the conversation.
 
     Methods:
         initiate_conversation(system, instruction, context=None, name=None):
             Initiate a conversation with a system setting and user instruction.
+
         add_messages(system, instruction, context=None, response=None, tool=None, name=None):
             Add messages to the conversation, including system setting, user instruction, and assistant response.
+
         change_system(system):
             Change the system setting in the conversation.
+
         keep_last_n_exchanges(n: int):
             Keep the last n exchanges in the conversation.
     """
@@ -48,11 +47,8 @@ class Conversation:
 
         Parameters:
             system (str): The system setting for the conversation.
-
             instruction (str): The user instruction to initiate the conversation.
-
             context (dict): Additional context for the conversation. Default is None.
-
             name (str): The name associated with the user. Default is None.
         """
         self.messages, self.responses = [], []
@@ -66,13 +62,10 @@ class Conversation:
 
         Parameters:
             system (str): The system setting for the message. Default is None.
-
             instruction (str): The instruction content for the message. Default is None.
-
             context (dict): Additional context for the message. Default is None.
-
             response (dict): The response content for the message. Default is None.
-
+            tool (dict): The tool information for the message. Default is None.
             name (str): The name associated with the message. Default is None.
         """
         msg = self.msg(system=system, instruction=instruction, context=context, 
@@ -101,5 +94,5 @@ class Conversation:
         ]
         if len(response_indices) >= n:
             first_index_to_keep = response_indices[-n] + 1
-            self.messages = self.messages[0] + self.messages[first_index_to_keep:]
+            self.messages = [self.system] + self.messages[first_index_to_keep:]
             

lionagi/core/flows.py

@@ -0,0 +1 @@
+# represents structured sessions

lionagi/core/instruction_sets.py

@@ -0,0 +1 @@
+# dynamically structured preconfigured instructions

lionagi/{session/message.py → core/messages.py}

@@ -1,7 +1,7 @@
 from datetime import datetime
 import json
-from ..utils.sys_util import create_id, l_call
-from ..utils.log_util import DataLogger
+from ..utils import create_id, lcall
+from ..schema import DataLogger
 
 
 class Message:
@@ -61,7 +61,7 @@ class Message:
 
             name (str): The name associated with the message. Default is None.
         """
-        if sum(l_call([system, instruction, response], bool)) > 1:
+        if sum(lcall([system, instruction, response], bool)) > 1:
             raise ValueError("Error: Message cannot have more than one role.")
         
         else: 
@@ -145,7 +145,7 @@ class Message:
                             context=context, response=response, name=name)
         return self.to_json()
     
-    def to_csv(self, dir=None, filename=None, verbose=True, timestamp=True, dir_exist_ok=True, file_exist_ok=False):
+    def to_csv(self, filename=None,dir=None,  verbose=True, timestamp=True, dir_exist_ok=True, file_exist_ok=False):
         """
         Save the message to a CSV file.
 
@@ -162,5 +162,5 @@ class Message:
 
             file_exist_ok (bool): Whether to allow the file to exist. Default is False.
         """
-        self._logger.to_csv(dir, filename, verbose, timestamp, dir_exist_ok, file_exist_ok)
+        self._logger.to_csv(filename,dir=dir, verbose=verbose, timestamp=timestamp, dir_exist_ok=dir_exist_ok, file_exist_ok=file_exist_ok)