lionagi 0.0.112__py3-none-any.whl → 0.0.113__py3-none-any.whl

lionagi/__init__.py

@@ -18,10 +18,10 @@ import logging
 from .version import __version__
 
 from .utils import *
-from .endpoint import *
-from .loader import *
-from .service_ import *
+from .schema import *
+from .structure import *
 from .core import *
+from .objs import *
 # from .datastore import *
 # from .structure import *
 

lionagi/bridge/__init__.py

@@ -0,0 +1,7 @@
+from .langchain import from_langchain
+from .llama_index import from_llama_index
+
+__all__ = [
+    "from_langchain",
+    "from_llama_index"
+]

lionagi/bridge/langchain.py

@@ -0,0 +1,131 @@
+from typing import Union, Callable, List, Dict, Any
+from ..schema.base_schema import T, DataNode
+from ..utils.sys_util import change_dict_key
+
+
+def from_langchain(lc_doc: Any) -> T:
+    """
+    Converts a langchain document into a DataNode object.
+
+    Parameters:
+        lc_doc (Any): The langchain document to be converted.
+
+    Returns:
+        DataNode: A DataNode object created from the langchain document.
+    """
+    info_json = lc_doc.to_json()
+    info_node = {'lc_id': info_json['id']}
+    info_node = {**info_node, **info_json['kwargs']}
+    return DataNode(**info_node)
+
+def to_langchain_document(datanode: T, **kwargs: Any) -> Any:
+    """
+    Converts a DataNode into a langchain Document.
+
+    Parameters:
+        datanode (DataNode): The DataNode to be converted.
+        
+        **kwargs: Additional keyword arguments to be included in the Document.
+
+    Returns:
+        Any: A langchain Document created from the DataNode.
+    """
+    from langchain.schema import Document
+
+    dnode = datanode.to_dict()
+    change_dict_key(dnode, old_key='content', new_key='page_content')
+    change_dict_key(dnode, old_key='lc_id', new_key='id_')
+    dnode = {**dnode, **kwargs}
+    return Document(**dnode)
+
+def langchain_loader(loader: Union[str, Callable], 
+                     loader_args: List[Any] = [], 
+                     loader_kwargs: Dict[str, Any] = {}) -> Any:
+    """
+    Loads data using a specified langchain loader.
+
+    Parameters:
+        loader (Union[str, Callable]): The name of the loader function or the loader function itself.
+        
+        loader_args (List[Any]): Positional arguments to pass to the loader function.
+        
+        loader_kwargs (Dict[str, Any]): Keyword arguments to pass to the loader function.
+
+    Returns:
+        Any: The data loaded by the loader function.
+
+    Raises:
+        ValueError: If the specified loader is invalid or if the loader fails to load data.
+    """
+    import langchain.document_loaders as document_loaders
+
+    try:
+        if isinstance(loader, str):
+            loader = getattr(document_loaders, loader)
+        else:
+            loader = loader
+    except Exception as e:
+        raise ValueError(f'Invalid loader: {loader}. Error: {e}')
+
+    try:
+        loader_obj = loader(*loader_args, **loader_kwargs)
+        data = loader_obj.load()
+        return data
+    except Exception as e:
+        raise ValueError(f'Failed to load. Error: {e}')
+
+def langchain_text_splitter(data: Union[str, List],
+                            splitter: Union[str, Callable], 
+                            splitter_args: List[Any] = [], 
+                            splitter_kwargs: Dict[str, Any] = {}) -> List[str]:
+
+    import langchain.text_splitter as text_splitter
+
+    try:
+        if isinstance(splitter, str):
+            splitter = getattr(text_splitter, splitter)
+        else:
+            splitter = splitter
+    except Exception as e:
+        raise ValueError(f'Invalid text splitter: {splitter}. Error: {e}')
+
+    try:
+        splitter_obj = splitter(*splitter_args, **splitter_kwargs)
+        if isinstance(data, str):
+            chunk = splitter_obj.split_text(data)
+        else:
+            chunk = splitter_obj.split_documents(data)
+        return chunk
+    except Exception as e:
+        raise ValueError(f'Failed to split. Error: {e}')
+
+# def langchain_code_splitter(doc: str,
+#                             language: str,
+#                             splitter_args: List[Any] = [],
+#                             splitter_kwargs: Dict[str, Any] = {}) -> List[Any]:
+#     """
+#     Splits code into smaller chunks using a RecursiveCharacterTextSplitter specific to a language.
+#
+#     Parameters:
+#         doc (str): The code document to be split.
+#         language (str): The programming language of the code.
+#         splitter_args (List[Any]): Positional arguments to pass to the splitter.
+#         splitter_kwargs (Dict[str, Any]): Keyword arguments to pass to the splitter.
+#
+#     Returns:
+#         List[Any]: A list of Documents, each representing a chunk of the original code.
+#
+#     Raises:
+#         ValueError: If the splitter fails to split the code document.
+#     """
+#     from langchain.text_splitter import RecursiveCharacterTextSplitter
+#
+#     try:
+#         splitter = RecursiveCharacterTextSplitter.from_language(
+#             language=language, *splitter_args, **splitter_kwargs
+#             )
+#         docs = splitter.create_documents([doc])
+#         return docs
+#     except Exception as e:
+#         raise ValueError(f'Failed to split. Error: {e}')
+#

lionagi/bridge/llama_index.py

@@ -0,0 +1,157 @@
+from typing import Union, Callable, List, Any, Dict
+from ..schema.base_schema import DataNode, T
+from ..utils.sys_util import change_dict_key
+
+
+def from_llama_index(llama_node: Any, **kwargs: Any) -> T:
+    """
+    Converts a Llama Index node into a DataNode object.
+
+    Parameters:
+        llama_node (Any): The Llama Index node to be converted.
+        
+        **kwargs: Additional keyword arguments for JSON serialization.
+
+    Returns:
+        DataNode: A DataNode object created from the Llama Index node.
+    """
+    llama_dict = llama_node.to_dict(**kwargs)
+    return DataNode.from_dict(llama_dict)
+
+def to_llama_index_textnode(datanode: T, **kwargs: Any) -> Any:
+    """
+    Converts a DataNode into a Llama Index TextNode.
+
+    Parameters:
+        datanode (DataNode): The DataNode to be converted.
+        
+        **kwargs: Additional keyword arguments to be included in the TextNode.
+
+    Returns:
+        TextNode: A Llama Index TextNode created from the DataNode.
+    """
+    # to llama_index textnode
+    from llama_index.schema import TextNode
+
+    dnode = datanode.to_dict()
+    change_dict_key(dnode, old_key='content', new_key='text')
+    change_dict_key(dnode, old_key='node_id', new_key='id_')
+    
+    dnode = {**dnode, **kwargs}
+    return TextNode.from_dict(dnode)
+
+def get_llama_reader(reader: Union[str, Callable]) -> Callable:
+    """
+    Gets a Llama Index reader function.
+
+    Parameters:
+        reader (Union[str, Callable]): The name of the reader function or the reader function itself.
+
+    Returns:
+        Callable: The Llama Index reader function.
+
+    Raises:
+        ValueError: If the specified reader is invalid.
+    """
+    try:
+        if isinstance(reader, str):
+            if reader == 'SimpleDirectoryReader':
+                from llama_index import SimpleDirectoryReader
+                return SimpleDirectoryReader
+            else:
+                from llama_index import download_loader
+                return download_loader(reader)
+        else:
+            return reader
+    except Exception as e:
+        raise ValueError(f'Invalid reader: {reader}, Error: {e}')
+
+def llama_index_reader(reader: Union[str, Callable], 
+                       reader_args: List[Any] = [], 
+                       reader_kwargs: Dict[str, Any] = {}, 
+                       load_data_args: List[Any] = [], 
+                       load_data_kwargs: Dict[str, Any] = {}) -> List[Any]:
+    """
+    Loads documents using a specified Llama Index reader.
+
+    Parameters:
+        reader (Union[str, Callable]): The name of the reader function or the reader function itself.
+        
+        reader_args (List[Any]): Positional arguments to pass to the reader function.
+        
+        reader_kwargs (Dict[str, Any]): Keyword arguments to pass to the reader function.
+        
+        load_data_args (List[Any]): Positional arguments for the load_data method.
+        
+        load_data_kwargs (Dict[str, Any]): Keyword arguments for the load_data method.
+
+    Returns:
+        List[Any]: A list of documents loaded by the reader.
+
+    Raises:
+        ValueError: If the specified reader is invalid or if the reader fails to load documents.
+    """
+    reader = get_llama_reader(reader)
+
+    try:
+        loader = reader(*reader_args, **reader_kwargs)
+        documents = loader.load_data(*load_data_args, **load_data_kwargs)
+        return documents
+    
+    except Exception as e:
+        raise ValueError(f'Failed to read. Error: {e}')
+
+def get_llama_parser(parser: Union[str, Callable]) -> Callable:
+    import llama_index.node_parser as node_parser
+    import llama_index.text_splitter as text_splitter
+
+    try:
+        return getattr(node_parser, parser)
+    except Exception as e1:
+        try:
+            if isinstance(parser, str):
+                return getattr(text_splitter, parser)
+            else:
+                return parser
+        except Exception as e2:
+            raise ValueError(f'Invalid node parser: {parser}. Error: {e1}, {e2}')
+
+
+def llama_index_node_parser(documents: List[Any], 
+                            parser: Union[str, Callable], 
+                            parser_args: List[Any] = [], 
+                            parser_kwargs: Dict[str, Any] = {},
+                            parsing_kwargs: Dict[str, Any] = {}) -> List[Any]:
+    """
+    Parses documents into nodes using a specified Llama Index node parser.
+
+    Parameters:
+        documents (List[Any]): The documents to parse.
+        
+        parser (Union[str, Callable]): The name of the parser function or the parser function itself.
+        
+        parser_args (List[Any]): Positional arguments to pass to the parser function.
+        
+        parser_kwargs (Dict[str, Any]): Keyword arguments to pass to the parser function.
+
+    Returns:
+        List[Any]: A list of nodes parsed from the documents.
+
+    Raises:
+        ValueError: If the specified parser is invalid or if the parser fails to parse the documents.
+    """
+    parser = get_llama_parser(parser)
+
+    try:
+        parser_obj = parser(*parser_args, **parser_kwargs)
+        nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
+        return nodes
+
+    except Exception as e1:
+        try:
+            parser_obj = parser.from_defaults(*parser_args, **parser_kwargs)
+            nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
+            return nodes
+        except Exception as e2:
+            raise ValueError(f'Failed to parse. Error: {e1}, {e2}')
+        

lionagi/configs/__init__.py

@@ -0,0 +1,7 @@
+from .oai_configs import oai_schema
+from .openrouter_config import openrouter_schema
+
+__all__ = [
+    "oai_schema",
+    "openrouter_schema"
+]

lionagi/configs/oai_configs.py

@@ -0,0 +1,49 @@
+oai_chat_llmconfig = {
+    "model": "gpt-4-1106-preview",
+    "frequency_penalty": 0,
+    "max_tokens": None,
+    "n": 1,
+    "presence_penalty": 0, 
+    "response_format": {"type": "text"}, 
+    "seed": None, 
+    "stop": None,
+    "stream": False,
+    "temperature": 0.7, 
+    "top_p": 1, 
+    "tools": None,
+    "tool_choice": "none", 
+    "user": None
+    }
+
+oai_chat_schema = {
+    "required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
+    "optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"],
+    "input": "messages",
+    "config": oai_chat_llmconfig
+    }
+
+oai_finetune_llmconfig = {
+    "model": "gpt-3.5-turbo",
+    "hyperparameters": {
+        "batch_size": "auto", 
+        "learning_rate_multiplier": "auto",
+        "n_epochs": "auto"
+    },
+    "suffix": None,
+    "training_file": None,
+    }
+
+oai_finetune_schema = {
+    "required" : ["model", "training_file"],
+    "optional": ["hyperparameters", "suffix", "validate_file"],
+    "input": ["training_file"],
+    "config": oai_finetune_llmconfig 
+}
+
+
+oai_schema = {
+    
+    "chat": oai_chat_schema,
+    "finetune": oai_finetune_schema
+    
+}

lionagi/configs/openrouter_config.py

@@ -0,0 +1,49 @@
+openrouter_chat_llmconfig = {
+    "model": "gpt-4-1106-preview",
+    "frequency_penalty": 0,
+    "max_tokens": None,
+    "n": 1,
+    "presence_penalty": 0, 
+    "response_format": {"type": "text"}, 
+    "seed": None, 
+    "stop": None,
+    "stream": False,
+    "temperature": 0.7, 
+    "top_p": 1, 
+    "tools": None,
+    "tool_choice": "none", 
+    "user": None
+    }
+
+openrouter_chat_schema = {
+    "required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
+    "optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"],
+    "input": "messages",
+    "config": openrouter_chat_llmconfig
+    }
+
+openrouter_finetune_llmconfig = {
+    "model": "gpt-3.5-turbo",
+    "hyperparameters": {
+        "batch_size": "auto", 
+        "learning_rate_multiplier": "auto",
+        "n_epochs": "auto"
+    },
+    "suffix": None,
+    "training_file": None,
+    }
+
+openrouter_finetune_schema = {
+    "required" : ["model", "training_file"],
+    "optional": ["hyperparameters", "suffix", "validate_file"],
+    "input": ["training_file"],
+    "config": openrouter_finetune_llmconfig
+}
+
+
+openrouter_schema = {
+
+    "chat": openrouter_chat_schema,
+    "finetune": openrouter_finetune_schema
+    
+}

lionagi/core/__init__.py

@@ -1,9 +1,15 @@
-from .messages import Message
+# from .messages import Response, Instruction, System
 from .conversations import Conversation
 from .sessions import Session
+
+# from .instruction_sets import InstructionSet
 # from .flows.flow import Flow
 
 
 __all__ = [
-    "Session", "Message", "Conversation"
+    # "Response",
+    # "Instruction",
+    # "System",
+    "Conversation",
+    "Session",              #"Flow", "InstructionSet"
 ]

lionagi/core/instruction_sets.py

@@ -1,3 +1 @@
-# TODO
-
-# a structured set of instruction nodes
+# dynamically structured preconfigured instructions

lionagi/core/messages.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 import json
-from ..utils.sys_utils import create_id, l_call
+from ..utils import create_id, lcall
 from ..schema import DataLogger
 
 
@@ -61,7 +61,7 @@ class Message:
 
             name (str): The name associated with the message. Default is None.
         """
-        if sum(l_call([system, instruction, response], bool)) > 1:
+        if sum(lcall([system, instruction, response], bool)) > 1:
             raise ValueError("Error: Message cannot have more than one role.")
         
         else: