lionagi 0.0.112__tar.gz → 0.0.113__tar.gz

{lionagi-0.0.112 → lionagi-0.0.113}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lionagi
-Version: 0.0.112
+Version: 0.0.113
 Summary: Towards automated general intelligence.
 Author: HaiyangLi
 Author-email: Haiyang Li <ocean@lionagi.ai>
@@ -220,7 +220,11 @@ Requires-Dist: python-dotenv==1.0.0
 Requires-Dist: tiktoken==0.5.1
 Requires-Dist: httpx==0.25.1
 
-![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935) [![Downloads](https://static.pepy.tech/badge/lionagi/week)](https://pepy.tech/project/lionagi)
+![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935)  ![Read the Docs](https://img.shields.io/readthedocs/lionagi)  ![PyPI - License](https://img.shields.io/pypi/l/lionagi?color=231fc935) ![PyPI - Downloads](https://img.shields.io/pypi/dm/lionagi?color=blue)
+
+
+
+
 
 [PyPI](https://pypi.org/project/lionagi/) | [Documentation](https://lionagi.readthedocs.io/en/latest/) | [Discord](https://discord.gg/7RGWqpSxze)
 
@@ -239,18 +243,20 @@ Install LionAGI with pip:
 ```bash
 pip install lionagi
 ```
-Download the `.env_template` file, input your OPENAI_API_KEY, save the file, rename as `.env` and put in your project's root directory. 
+Download the `.env_template` file, input your appropriate `API_KEY`, save the file, rename as `.env` and put in your project's root directory. 
+by default we use `OPENAI_API_KEY`.
 
-### Features
 
-- Robust performance
-- Efficient data operations for reading, chunking, binning, writing, storing and managing data.
-- Fast interaction with LLM services like OpenAI with **configurable rate limiting concurrent API calls** for maximum throughput. 
-- Create a production ready LLM application **in hours**. Intuitive workflow management to streamline the process from idea to market.
-- (Work In Progress): verstile intergration with most API and local LLM services.  
 
+### Features
+- Create a production ready LLM application **in hours**, with more than 100 models to choose from
+- written in pure python, minimum dependency `aiohttp`, `python-dotenv`, `tiktoken`, `pydantic`
+- Efficient and verstile data operations for reading, chunking, binning, writing, storing data with built-in support for `langchain` and `llamaindex`
+- Unified interface with any LLM provider, API or local
+  - Fast and **concurrent** API call with **configurable rate limit**
+  - (Work In Progress) support for hundreds of models both API and local
 ---
-LionAGI is designed to be async only, please check python official documentation on how `async` work: [here](https://docs.python.org/3/library/asyncio.html)
+LionAGI is designed to be `asynchronous` only, please check python official documentation on how `async` work: [here](https://docs.python.org/3/library/asyncio.html)
 
 
 **Notice**: 
@@ -271,11 +277,11 @@ import lionagi as li
 system = "You are a helpful assistant designed to perform calculations."
 instruction = {"Addition":"Add the two numbers together i.e. x+y"}
 context = {"x": 10, "y": 5}
+```
 
-# Initialize a session with a system message
+```python
+# in interactive environment (.ipynb for example)
 calculator = li.Session(system=system)
-
-# run a LLM API call
 result = await calculator.initiate(instruction=instruction,
                                    context=context,
                                    model="gpt-4-1106-preview")
@@ -283,6 +289,24 @@ result = await calculator.initiate(instruction=instruction,
 print(f"Calculation Result: {result}")
 ```
 
+```python
+# or otherwise, you can use
+import asyncio
+from dotenv import loadenv
+
+load_dotenv()
+
+async def main():
+    calculator = li.Session(system=system)
+    result = await calculator.initiate(instruction=instruction,
+                                       context=context, 
+                                       model="gpt-4-1106-preview")
+    print(f"Calculation Result: {result}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
 Visit our notebooks for our examples. 
 
 ### Community

{lionagi-0.0.112 → lionagi-0.0.113}/README.md

@@ -1,4 +1,8 @@
-![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935) [![Downloads](https://static.pepy.tech/badge/lionagi/week)](https://pepy.tech/project/lionagi)
+![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935)  ![Read the Docs](https://img.shields.io/readthedocs/lionagi)  ![PyPI - License](https://img.shields.io/pypi/l/lionagi?color=231fc935) ![PyPI - Downloads](https://img.shields.io/pypi/dm/lionagi?color=blue)
+
+
+
+
 
 [PyPI](https://pypi.org/project/lionagi/) | [Documentation](https://lionagi.readthedocs.io/en/latest/) | [Discord](https://discord.gg/7RGWqpSxze)
 
@@ -17,18 +21,20 @@ Install LionAGI with pip:
 ```bash
 pip install lionagi
 ```
-Download the `.env_template` file, input your OPENAI_API_KEY, save the file, rename as `.env` and put in your project's root directory. 
+Download the `.env_template` file, input your appropriate `API_KEY`, save the file, rename as `.env` and put in your project's root directory. 
+by default we use `OPENAI_API_KEY`.
 
-### Features
 
-- Robust performance
-- Efficient data operations for reading, chunking, binning, writing, storing and managing data.
-- Fast interaction with LLM services like OpenAI with **configurable rate limiting concurrent API calls** for maximum throughput. 
-- Create a production ready LLM application **in hours**. Intuitive workflow management to streamline the process from idea to market.
-- (Work In Progress): verstile intergration with most API and local LLM services.  
 
+### Features
+- Create a production ready LLM application **in hours**, with more than 100 models to choose from
+- written in pure python, minimum dependency `aiohttp`, `python-dotenv`, `tiktoken`, `pydantic`
+- Efficient and verstile data operations for reading, chunking, binning, writing, storing data with built-in support for `langchain` and `llamaindex`
+- Unified interface with any LLM provider, API or local
+  - Fast and **concurrent** API call with **configurable rate limit**
+  - (Work In Progress) support for hundreds of models both API and local
 ---
-LionAGI is designed to be async only, please check python official documentation on how `async` work: [here](https://docs.python.org/3/library/asyncio.html)
+LionAGI is designed to be `asynchronous` only, please check python official documentation on how `async` work: [here](https://docs.python.org/3/library/asyncio.html)
 
 
 **Notice**: 
@@ -49,11 +55,11 @@ import lionagi as li
 system = "You are a helpful assistant designed to perform calculations."
 instruction = {"Addition":"Add the two numbers together i.e. x+y"}
 context = {"x": 10, "y": 5}
+```
 
-# Initialize a session with a system message
+```python
+# in interactive environment (.ipynb for example)
 calculator = li.Session(system=system)
-
-# run a LLM API call
 result = await calculator.initiate(instruction=instruction,
                                    context=context,
                                    model="gpt-4-1106-preview")
@@ -61,6 +67,24 @@ result = await calculator.initiate(instruction=instruction,
 print(f"Calculation Result: {result}")
 ```
 
+```python
+# or otherwise, you can use
+import asyncio
+from dotenv import loadenv
+
+load_dotenv()
+
+async def main():
+    calculator = li.Session(system=system)
+    result = await calculator.initiate(instruction=instruction,
+                                       context=context, 
+                                       model="gpt-4-1106-preview")
+    print(f"Calculation Result: {result}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
 Visit our notebooks for our examples. 
 
 ### Community

{lionagi-0.0.112 → lionagi-0.0.113}/lionagi/__init__.py

@@ -18,10 +18,10 @@ import logging
 from .version import __version__
 
 from .utils import *
-from .endpoint import *
-from .loader import *
-from .service_ import *
+from .schema import *
+from .structure import *
 from .core import *
+from .objs import *
 # from .datastore import *
 # from .structure import *
 

lionagi-0.0.113/lionagi/bridge/__init__.py

@@ -0,0 +1,7 @@
+from .langchain import from_langchain
+from .llama_index import from_llama_index
+
+__all__ = [
+    "from_langchain",
+    "from_llama_index"
+]

lionagi-0.0.113/lionagi/bridge/langchain.py

@@ -0,0 +1,131 @@
+from typing import Union, Callable, List, Dict, Any
+from ..schema.base_schema import T, DataNode
+from ..utils.sys_util import change_dict_key
+
+
+def from_langchain(lc_doc: Any) -> T:
+    """
+    Converts a langchain document into a DataNode object.
+
+    Parameters:
+        lc_doc (Any): The langchain document to be converted.
+
+    Returns:
+        DataNode: A DataNode object created from the langchain document.
+    """
+    info_json = lc_doc.to_json()
+    info_node = {'lc_id': info_json['id']}
+    info_node = {**info_node, **info_json['kwargs']}
+    return DataNode(**info_node)
+
+def to_langchain_document(datanode: T, **kwargs: Any) -> Any:
+    """
+    Converts a DataNode into a langchain Document.
+
+    Parameters:
+        datanode (DataNode): The DataNode to be converted.
+        
+        **kwargs: Additional keyword arguments to be included in the Document.
+
+    Returns:
+        Any: A langchain Document created from the DataNode.
+    """
+    from langchain.schema import Document
+
+    dnode = datanode.to_dict()
+    change_dict_key(dnode, old_key='content', new_key='page_content')
+    change_dict_key(dnode, old_key='lc_id', new_key='id_')
+    dnode = {**dnode, **kwargs}
+    return Document(**dnode)
+
+def langchain_loader(loader: Union[str, Callable], 
+                     loader_args: List[Any] = [], 
+                     loader_kwargs: Dict[str, Any] = {}) -> Any:
+    """
+    Loads data using a specified langchain loader.
+
+    Parameters:
+        loader (Union[str, Callable]): The name of the loader function or the loader function itself.
+        
+        loader_args (List[Any]): Positional arguments to pass to the loader function.
+        
+        loader_kwargs (Dict[str, Any]): Keyword arguments to pass to the loader function.
+
+    Returns:
+        Any: The data loaded by the loader function.
+
+    Raises:
+        ValueError: If the specified loader is invalid or if the loader fails to load data.
+    """
+    import langchain.document_loaders as document_loaders
+
+    try:
+        if isinstance(loader, str):
+            loader = getattr(document_loaders, loader)
+        else:
+            loader = loader
+    except Exception as e:
+        raise ValueError(f'Invalid loader: {loader}. Error: {e}')
+
+    try:
+        loader_obj = loader(*loader_args, **loader_kwargs)
+        data = loader_obj.load()
+        return data
+    except Exception as e:
+        raise ValueError(f'Failed to load. Error: {e}')
+
+def langchain_text_splitter(data: Union[str, List],
+                            splitter: Union[str, Callable], 
+                            splitter_args: List[Any] = [], 
+                            splitter_kwargs: Dict[str, Any] = {}) -> List[str]:
+
+    import langchain.text_splitter as text_splitter
+
+    try:
+        if isinstance(splitter, str):
+            splitter = getattr(text_splitter, splitter)
+        else:
+            splitter = splitter
+    except Exception as e:
+        raise ValueError(f'Invalid text splitter: {splitter}. Error: {e}')
+
+    try:
+        splitter_obj = splitter(*splitter_args, **splitter_kwargs)
+        if isinstance(data, str):
+            chunk = splitter_obj.split_text(data)
+        else:
+            chunk = splitter_obj.split_documents(data)
+        return chunk
+    except Exception as e:
+        raise ValueError(f'Failed to split. Error: {e}')
+
+# def langchain_code_splitter(doc: str,
+#                             language: str,
+#                             splitter_args: List[Any] = [],
+#                             splitter_kwargs: Dict[str, Any] = {}) -> List[Any]:
+#     """
+#     Splits code into smaller chunks using a RecursiveCharacterTextSplitter specific to a language.
+#
+#     Parameters:
+#         doc (str): The code document to be split.
+#         language (str): The programming language of the code.
+#         splitter_args (List[Any]): Positional arguments to pass to the splitter.
+#         splitter_kwargs (Dict[str, Any]): Keyword arguments to pass to the splitter.
+#
+#     Returns:
+#         List[Any]: A list of Documents, each representing a chunk of the original code.
+#
+#     Raises:
+#         ValueError: If the splitter fails to split the code document.
+#     """
+#     from langchain.text_splitter import RecursiveCharacterTextSplitter
+#
+#     try:
+#         splitter = RecursiveCharacterTextSplitter.from_language(
+#             language=language, *splitter_args, **splitter_kwargs
+#             )
+#         docs = splitter.create_documents([doc])
+#         return docs
+#     except Exception as e:
+#         raise ValueError(f'Failed to split. Error: {e}')
+#

lionagi-0.0.113/lionagi/bridge/llama_index.py

@@ -0,0 +1,157 @@
+from typing import Union, Callable, List, Any, Dict
+from ..schema.base_schema import DataNode, T
+from ..utils.sys_util import change_dict_key
+
+
+def from_llama_index(llama_node: Any, **kwargs: Any) -> T:
+    """
+    Converts a Llama Index node into a DataNode object.
+
+    Parameters:
+        llama_node (Any): The Llama Index node to be converted.
+        
+        **kwargs: Additional keyword arguments for JSON serialization.
+
+    Returns:
+        DataNode: A DataNode object created from the Llama Index node.
+    """
+    llama_dict = llama_node.to_dict(**kwargs)
+    return DataNode.from_dict(llama_dict)
+
+def to_llama_index_textnode(datanode: T, **kwargs: Any) -> Any:
+    """
+    Converts a DataNode into a Llama Index TextNode.
+
+    Parameters:
+        datanode (DataNode): The DataNode to be converted.
+        
+        **kwargs: Additional keyword arguments to be included in the TextNode.
+
+    Returns:
+        TextNode: A Llama Index TextNode created from the DataNode.
+    """
+    # to llama_index textnode
+    from llama_index.schema import TextNode
+
+    dnode = datanode.to_dict()
+    change_dict_key(dnode, old_key='content', new_key='text')
+    change_dict_key(dnode, old_key='node_id', new_key='id_')
+    
+    dnode = {**dnode, **kwargs}
+    return TextNode.from_dict(dnode)
+
+def get_llama_reader(reader: Union[str, Callable]) -> Callable:
+    """
+    Gets a Llama Index reader function.
+
+    Parameters:
+        reader (Union[str, Callable]): The name of the reader function or the reader function itself.
+
+    Returns:
+        Callable: The Llama Index reader function.
+
+    Raises:
+        ValueError: If the specified reader is invalid.
+    """
+    try:
+        if isinstance(reader, str):
+            if reader == 'SimpleDirectoryReader':
+                from llama_index import SimpleDirectoryReader
+                return SimpleDirectoryReader
+            else:
+                from llama_index import download_loader
+                return download_loader(reader)
+        else:
+            return reader
+    except Exception as e:
+        raise ValueError(f'Invalid reader: {reader}, Error: {e}')
+
+def llama_index_reader(reader: Union[str, Callable], 
+                       reader_args: List[Any] = [], 
+                       reader_kwargs: Dict[str, Any] = {}, 
+                       load_data_args: List[Any] = [], 
+                       load_data_kwargs: Dict[str, Any] = {}) -> List[Any]:
+    """
+    Loads documents using a specified Llama Index reader.
+
+    Parameters:
+        reader (Union[str, Callable]): The name of the reader function or the reader function itself.
+        
+        reader_args (List[Any]): Positional arguments to pass to the reader function.
+        
+        reader_kwargs (Dict[str, Any]): Keyword arguments to pass to the reader function.
+        
+        load_data_args (List[Any]): Positional arguments for the load_data method.
+        
+        load_data_kwargs (Dict[str, Any]): Keyword arguments for the load_data method.
+
+    Returns:
+        List[Any]: A list of documents loaded by the reader.
+
+    Raises:
+        ValueError: If the specified reader is invalid or if the reader fails to load documents.
+    """
+    reader = get_llama_reader(reader)
+
+    try:
+        loader = reader(*reader_args, **reader_kwargs)
+        documents = loader.load_data(*load_data_args, **load_data_kwargs)
+        return documents
+    
+    except Exception as e:
+        raise ValueError(f'Failed to read. Error: {e}')
+
+def get_llama_parser(parser: Union[str, Callable]) -> Callable:
+    import llama_index.node_parser as node_parser
+    import llama_index.text_splitter as text_splitter
+
+    try:
+        return getattr(node_parser, parser)
+    except Exception as e1:
+        try:
+            if isinstance(parser, str):
+                return getattr(text_splitter, parser)
+            else:
+                return parser
+        except Exception as e2:
+            raise ValueError(f'Invalid node parser: {parser}. Error: {e1}, {e2}')
+
+
+def llama_index_node_parser(documents: List[Any], 
+                            parser: Union[str, Callable], 
+                            parser_args: List[Any] = [], 
+                            parser_kwargs: Dict[str, Any] = {},
+                            parsing_kwargs: Dict[str, Any] = {}) -> List[Any]:
+    """
+    Parses documents into nodes using a specified Llama Index node parser.
+
+    Parameters:
+        documents (List[Any]): The documents to parse.
+        
+        parser (Union[str, Callable]): The name of the parser function or the parser function itself.
+        
+        parser_args (List[Any]): Positional arguments to pass to the parser function.
+        
+        parser_kwargs (Dict[str, Any]): Keyword arguments to pass to the parser function.
+
+    Returns:
+        List[Any]: A list of nodes parsed from the documents.
+
+    Raises:
+        ValueError: If the specified parser is invalid or if the parser fails to parse the documents.
+    """
+    parser = get_llama_parser(parser)
+
+    try:
+        parser_obj = parser(*parser_args, **parser_kwargs)
+        nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
+        return nodes
+
+    except Exception as e1:
+        try:
+            parser_obj = parser.from_defaults(*parser_args, **parser_kwargs)
+            nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
+            return nodes
+        except Exception as e2:
+            raise ValueError(f'Failed to parse. Error: {e1}, {e2}')
+        

lionagi-0.0.113/lionagi/configs/__init__.py

@@ -0,0 +1,7 @@
+from .oai_configs import oai_schema
+from .openrouter_config import openrouter_schema
+
+__all__ = [
+    "oai_schema",
+    "openrouter_schema"
+]

lionagi-0.0.113/lionagi/configs/oai_configs.py

@@ -0,0 +1,49 @@
+oai_chat_llmconfig = {
+    "model": "gpt-4-1106-preview",
+    "frequency_penalty": 0,
+    "max_tokens": None,
+    "n": 1,
+    "presence_penalty": 0, 
+    "response_format": {"type": "text"}, 
+    "seed": None, 
+    "stop": None,
+    "stream": False,
+    "temperature": 0.7, 
+    "top_p": 1, 
+    "tools": None,
+    "tool_choice": "none", 
+    "user": None
+    }
+
+oai_chat_schema = {
+    "required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
+    "optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"],
+    "input": "messages",
+    "config": oai_chat_llmconfig
+    }
+
+oai_finetune_llmconfig = {
+    "model": "gpt-3.5-turbo",
+    "hyperparameters": {
+        "batch_size": "auto", 
+        "learning_rate_multiplier": "auto",
+        "n_epochs": "auto"
+    },
+    "suffix": None,
+    "training_file": None,
+    }
+
+oai_finetune_schema = {
+    "required" : ["model", "training_file"],
+    "optional": ["hyperparameters", "suffix", "validate_file"],
+    "input": ["training_file"],
+    "config": oai_finetune_llmconfig 
+}
+
+
+oai_schema = {
+    
+    "chat": oai_chat_schema,
+    "finetune": oai_finetune_schema
+    
+}

lionagi-0.0.113/lionagi/configs/openrouter_config.py

@@ -0,0 +1,49 @@
+openrouter_chat_llmconfig = {
+    "model": "gpt-4-1106-preview",
+    "frequency_penalty": 0,
+    "max_tokens": None,
+    "n": 1,
+    "presence_penalty": 0, 
+    "response_format": {"type": "text"}, 
+    "seed": None, 
+    "stop": None,
+    "stream": False,
+    "temperature": 0.7, 
+    "top_p": 1, 
+    "tools": None,
+    "tool_choice": "none", 
+    "user": None
+    }
+
+openrouter_chat_schema = {
+    "required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
+    "optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"],
+    "input": "messages",
+    "config": openrouter_chat_llmconfig
+    }
+
+openrouter_finetune_llmconfig = {
+    "model": "gpt-3.5-turbo",
+    "hyperparameters": {
+        "batch_size": "auto", 
+        "learning_rate_multiplier": "auto",
+        "n_epochs": "auto"
+    },
+    "suffix": None,
+    "training_file": None,
+    }
+
+openrouter_finetune_schema = {
+    "required" : ["model", "training_file"],
+    "optional": ["hyperparameters", "suffix", "validate_file"],
+    "input": ["training_file"],
+    "config": openrouter_finetune_llmconfig
+}
+
+
+openrouter_schema = {
+
+    "chat": openrouter_chat_schema,
+    "finetune": openrouter_finetune_schema
+    
+}

lionagi-0.0.113/lionagi/core/__init__.py

@@ -0,0 +1,15 @@
+# from .messages import Response, Instruction, System
+from .conversations import Conversation
+from .sessions import Session
+
+# from .instruction_sets import InstructionSet
+# from .flows.flow import Flow
+
+
+__all__ = [
+    # "Response",
+    # "Instruction",
+    # "System",
+    "Conversation",
+    "Session",              #"Flow", "InstructionSet"
+]

lionagi-0.0.113/lionagi/core/instruction_sets.py

@@ -0,0 +1 @@
+# dynamically structured preconfigured instructions

{lionagi-0.0.112 → lionagi-0.0.113}/lionagi/core/messages.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 import json
-from ..utils.sys_utils import create_id, l_call
+from ..utils import create_id, lcall
 from ..schema import DataLogger
 
 
@@ -61,7 +61,7 @@ class Message:
 
             name (str): The name associated with the message. Default is None.
         """
-        if sum(l_call([system, instruction, response], bool)) > 1:
+        if sum(lcall([system, instruction, response], bool)) > 1:
             raise ValueError("Error: Message cannot have more than one role.")
         
         else: