lionagi 0.1.2__py3-none-any.whl → 0.2.1__py3-none-any.whl

lionagi/integrations/chunker/chunk.py

@@ -1,6 +1,8 @@
 from typing import Union, Callable
 
 from lionagi.libs import func_call
+from lionagi.libs.ln_convert import to_list
+from lionagi.core.collections import pile
 from lionagi.core.generic import Node
 from ..bridge.langchain_.langchain_bridge import LangchainBridge
 from ..bridge.llamaindex_.llama_index_bridge import LlamaIndexBridge
@@ -10,7 +12,20 @@ from ..loader.load_util import ChunkerType, file_to_chunks, _datanode_parser
 
 
 def datanodes_convert(documents, chunker_type):
+    """
+    Converts documents to the specified chunker type.
 
+    Args:
+        documents (list): List of documents to be converted.
+        chunker_type (ChunkerType): The type of chunker to convert the documents to.
+
+    Returns:
+        list: The converted documents.
+
+    Example usage:
+        >>> documents = [Node(...), Node(...)]
+        >>> converted_docs = datanodes_convert(documents, ChunkerType.LLAMAINDEX)
+    """
     for i in range(len(documents)):
         if type(documents[i]) == Node:
             if chunker_type == ChunkerType.LLAMAINDEX:
@@ -21,25 +36,71 @@ def datanodes_convert(documents, chunker_type):
 
 
 def text_chunker(documents, args, kwargs):
+    """
+    Chunks text documents into smaller pieces.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        args (tuple): Positional arguments for the chunking function.
+        kwargs (dict): Keyword arguments for the chunking function.
+
+    Returns:
+        pile: A pile of chunked Node instances.
+
+    Example usage:
+        >>> documents = [Node(...), Node(...)]
+        >>> chunked_docs = text_chunker(documents, args, kwargs)
+    """
 
     def chunk_node(node):
         chunks = file_to_chunks(node.to_dict(), *args, **kwargs)
-        func_call.lcall(chunks, lambda chunk: chunk.pop("node_id"))
+        func_call.lcall(chunks, lambda chunk: chunk.pop("ln_id"))
         return [Node.from_obj({**chunk}) for chunk in chunks]
 
-    return [chunk_node(doc) for doc in documents]
+    a = to_list([chunk_node(doc) for doc in documents], flatten=True, dropna=True)
+    return pile(a)
 
 
 def chunk(
-    documents,
-    chunker,
+    docs,
+    field: str = "content",
+    chunk_size: int = 1500,
+    overlap: float = 0.1,
+    threshold: int = 200,
+    chunker="text_chunker",
     chunker_type=ChunkerType.PLAIN,
     chunker_args=None,
     chunker_kwargs=None,
     chunking_kwargs=None,
     documents_convert_func=None,
-    to_datanode: bool | Callable = True,
+    to_lion: bool | Callable = True,
 ):
+    """
+    Chunks documents using the specified chunker.
+
+    Args:
+        docs (list): List of documents to be chunked.
+        field (str, optional): The field to chunk. Defaults to "content".
+        chunk_size (int, optional): The size of each chunk. Defaults to 1500.
+        overlap (float, optional): The overlap between chunks. Defaults to 0.1.
+        threshold (int, optional): The threshold for chunking. Defaults to 200.
+        chunker (str, optional): The chunker function or its name. Defaults to "text_chunker".
+        chunker_type (ChunkerType, optional): The type of chunker to use. Defaults to ChunkerType.PLAIN.
+        chunker_args (list, optional): Positional arguments for the chunker function. Defaults to None.
+        chunker_kwargs (dict, optional): Keyword arguments for the chunker function. Defaults to None.
+        chunking_kwargs (dict, optional): Additional keyword arguments for chunking. Defaults to None.
+        documents_convert_func (Callable, optional): Function to convert documents. Defaults to None.
+        to_lion (bool | Callable, optional): Whether to convert the data to Node instances or a custom parser. Defaults to True.
+
+    Returns:
+        pile: A pile of chunked Node instances.
+
+    Raises:
+        ValueError: If the chunker_type is not supported.
+
+    Example usage:
+        >>> chunked_docs = chunk(docs, field='text', chunk_size=1000, overlap=0.2)
+    """
 
     if chunker_args is None:
         chunker_args = []
@@ -49,38 +110,42 @@ def chunk(
         chunking_kwargs = {}
 
     if chunker_type == ChunkerType.PLAIN:
+        chunker_kwargs["field"] = field
+        chunker_kwargs["chunk_size"] = chunk_size
+        chunker_kwargs["overlap"] = overlap
+        chunker_kwargs["threshold"] = threshold
         return chunk_funcs[ChunkerType.PLAIN](
-            documents, chunker, chunker_args, chunker_kwargs
+            docs, chunker, chunker_args, chunker_kwargs
         )
 
     elif chunker_type == ChunkerType.LANGCHAIN:
         return chunk_funcs[ChunkerType.LANGCHAIN](
-            documents,
+            docs,
             documents_convert_func,
             chunker,
             chunker_args,
             chunker_kwargs,
-            to_datanode,
+            to_lion,
         )
 
     elif chunker_type == ChunkerType.LLAMAINDEX:
         return chunk_funcs[ChunkerType.LLAMAINDEX](
-            documents,
+            docs,
             documents_convert_func,
             chunker,
             chunker_args,
             chunker_kwargs,
-            to_datanode,
+            to_lion,
         )
 
     elif chunker_type == ChunkerType.SELFDEFINED:
         return chunk_funcs[ChunkerType.SELFDEFINED](
-            documents,
+            docs,
             chunker,
             chunker_args,
             chunker_kwargs,
             chunking_kwargs,
-            to_datanode,
+            to_lion,
         )
 
     else:
@@ -95,8 +160,28 @@ def _self_defined_chunker(
     chunker_args,
     chunker_kwargs,
     chunking_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Chunks documents using a self-defined chunker.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        chunker (str | Callable): The chunker function or its name.
+        chunker_args (list): Positional arguments for the chunker function.
+        chunker_kwargs (dict): Keyword arguments for the chunker function.
+        chunking_kwargs (dict): Additional keyword arguments for chunking.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of chunked Node instances or custom parsed nodes.
+
+    Raises:
+        ValueError: If the self-defined chunker is not valid.
+
+    Example usage:
+        >>> chunked_docs = _self_defined_chunker(docs, custom_chunker, ['arg1'], {'key': 'value'}, {}, custom_parser)
+    """
     try:
         splitter = chunker(*chunker_args, **chunker_kwargs)
         nodes = splitter.split(documents, **chunking_kwargs)
@@ -105,10 +190,10 @@ def _self_defined_chunker(
             f"Self defined chunker {chunker} is not valid. Error: {e}"
         ) from e
 
-    if isinstance(to_datanode, bool) and to_datanode is True:
+    if isinstance(to_lion, bool) and to_lion is True:
         raise ValueError("Please define a valid parser to Node.")
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
@@ -118,18 +203,35 @@ def _llama_index_chunker(
     chunker,
     chunker_args,
     chunker_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Chunks documents using a LlamaIndex chunker.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        documents_convert_func (Callable): Function to convert documents.
+        chunker (str | Callable): The chunker function or its name.
+        chunker_args (list): Positional arguments for the chunker function.
+        chunker_kwargs (dict): Keyword arguments for the chunker function.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of chunked Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> chunked_docs = _llama_index_chunker(docs, convert_func, llama_chunker, ['arg1'], {'key': 'value'}, True)
+    """
     if documents_convert_func:
         documents = documents_convert_func(documents, "llama_index")
     nodes = LlamaIndexBridge.llama_index_parse_node(
         documents, chunker, chunker_args, chunker_kwargs
     )
 
-    if isinstance(to_datanode, bool) and to_datanode is True:
+    if isinstance(to_lion, bool) and to_lion is True:
         nodes = [Node.from_llama_index(i) for i in nodes]
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
@@ -139,24 +241,59 @@ def _langchain_chunker(
     chunker,
     chunker_args,
     chunker_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Chunks documents using a Langchain chunker.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        documents_convert_func (Callable): Function to convert documents.
+        chunker (str | Callable): The chunker function or its name.
+        chunker_args (list): Positional arguments for the chunker function.
+        chunker_kwargs (dict): Keyword arguments for the chunker function.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of chunked Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> chunked_docs = _langchain_chunker(docs, convert_func, langchain_chunker, ['arg1'], {'key': 'value'}, True)
+    """
     if documents_convert_func:
         documents = documents_convert_func(documents, "langchain")
     nodes = LangchainBridge.langchain_text_splitter(
         documents, chunker, chunker_args, chunker_kwargs
     )
-    if isinstance(to_datanode, bool) and to_datanode is True:
+    if isinstance(to_lion, bool) and to_lion is True:
         if isinstance(documents, str):
             nodes = [Node(content=i) for i in nodes]
         else:
             nodes = [Node.from_langchain(i) for i in nodes]
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
 def _plain_chunker(documents, chunker, chunker_args, chunker_kwargs):
+    """
+    Chunks documents using a plain chunker.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        chunker (str | Callable): The chunker function or its name.
+        chunker_args (list): Positional arguments for the chunker function.
+        chunker_kwargs (dict): Keyword arguments for the chunker function.
+
+    Returns:
+        pile: A pile of chunked Node instances.
+
+    Raises:
+        ValueError: If the chunker is not supported.
+
+    Example usage:
+        >>> chunked_docs = _plain_chunker(docs, 'text_chunker', ['arg1'], {'key': 'value'})
+    """
     try:
         if chunker == "text_chunker":
             chunker = text_chunker

lionagi/integrations/config/oai_configs.py

@@ -1,8 +1,10 @@
 # Default configs for the OpenAI API
 
+API_key_schema = ("OPENAI_API_KEY",)
+
 # ChatCompletion
 oai_chat_llmconfig = {
-    "model": "gpt-4-turbo",
+    "model": "gpt-4o",
     "frequency_penalty": 0,
     "max_tokens": None,
     "n": 1,
@@ -11,11 +13,13 @@ oai_chat_llmconfig = {
     "seed": None,
     "stop": None,
     "stream": False,
-    "temperature": 0.7,
+    "temperature": 0.1,
     "top_p": 1,
     "tools": None,
     "tool_choice": "none",
     "user": None,
+    "logprobs": False,
+    "top_logprobs": None,
 }
 
 oai_chat_schema = {
@@ -36,9 +40,16 @@ oai_chat_schema = {
         "tool_choice",
         "user",
         "max_tokens",
+        "logprobs",
+        "top_logprobs",
     ],
     "input_": "messages",
     "config": oai_chat_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 128_000,
+    "interval_tokens": 1_000_000,
+    "interval_requests": 1_000,
+    "interval": 60,
 }
 
 # Finetune
@@ -111,8 +122,26 @@ oai_audio_translations_schema = {
     "config": oai_audio_translations_llmconfig,
 }
 
-# images
+# embeddings
 
+oai_embeddings_llmconfig = {
+    "model": "text-embedding-ada-002",
+    "encoding_format": "float",
+    "user": None,
+    "dimensions": None,
+}
+
+oai_embeddings_schema = {
+    "required": ["model", "encoding_format"],
+    "optional": ["user", "dimensions"],
+    "input_": "input",
+    "config": oai_embeddings_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 8192,
+    "interval_tokens": 1_000_000,
+    "interval_requests": 1_000,
+    "interval": 60,
+}
 
 oai_schema = {
     "chat/completions": oai_chat_schema,
@@ -120,4 +149,6 @@ oai_schema = {
     "audio_speech": oai_audio_speech_schema,
     "audio_transcriptions": oai_audio_transcriptions_schema,
     "audio_translations": oai_audio_translations_schema,
+    "API_key_schema": API_key_schema,
+    "embeddings": oai_embeddings_schema,
 }

lionagi/integrations/config/openrouter_configs.py

@@ -1,5 +1,7 @@
+API_key_schema = ("OPENROUTER_API_KEY",)
+
 openrouter_chat_llmconfig = {
-    "model": "gpt-4-turbo",
+    "model": "gpt-4o",
     "frequency_penalty": 0,
     "max_tokens": None,
     "num": 1,
@@ -8,11 +10,13 @@ openrouter_chat_llmconfig = {
     "seed": None,
     "stop": None,
     "stream": False,
-    "temperature": 0.7,
+    "temperature": 0.1,
     "top_p": 1,
     "tools": None,
     "tool_choice": "none",
     "user": None,
+    "logprobs": False,
+    "top_logprobs": None,
 }
 
 openrouter_chat_schema = {
@@ -33,9 +37,16 @@ openrouter_chat_schema = {
         "tool_choice",
         "user",
         "max_tokens",
+        "logprobs",
+        "top_logprobs",
     ],
     "input_": "messages",
     "config": openrouter_chat_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 128_000,
+    "interval_tokens": 10_000,
+    "interval_requests": 100,
+    "interval": 60,
 }
 
 openrouter_finetune_llmconfig = {
@@ -59,4 +70,5 @@ openrouter_finetune_schema = {
 openrouter_schema = {
     "chat/completions": openrouter_chat_schema,
     "finetune": openrouter_finetune_schema,
+    "API_key_schema": API_key_schema,
 }

lionagi/integrations/loader/load.py

@@ -1,6 +1,7 @@
 from typing import Callable
 
 from lionagi.core.generic import Node
+from lionagi.core.collections import pile
 from ..bridge.langchain_.langchain_bridge import LangchainBridge
 from ..bridge.llamaindex_.llama_index_bridge import LlamaIndexBridge
 
@@ -27,18 +28,43 @@ def text_reader(args, kwargs):
 
 
 def load(
-    reader: str | Callable = "SimpleDirectoryReader",
+    reader: str | Callable = "text_reader",
     input_dir=None,
     input_files=None,
     recursive: bool = False,
     required_exts: list[str] = None,
-    reader_type=ReaderType.LLAMAINDEX,
+    reader_type=ReaderType.PLAIN,
     reader_args=None,
     reader_kwargs=None,
     load_args=None,
     load_kwargs=None,
-    to_datanode: bool | Callable = True,
+    to_lion: bool | Callable = True,
 ):
+    """
+    Loads data using the specified reader and converts it to Node instances.
+
+    Args:
+        reader (str | Callable): The reader function or its name. Defaults to "text_reader".
+        input_dir (str, optional): The directory to read files from. Defaults to None.
+        input_files (list[str], optional): Specific files to read. Defaults to None.
+        recursive (bool, optional): Whether to read files recursively. Defaults to False.
+        required_exts (list[str], optional): List of required file extensions. Defaults to None.
+        reader_type (ReaderType, optional): The type of reader to use. Defaults to ReaderType.PLAIN.
+        reader_args (list, optional): Positional arguments for the reader function. Defaults to None.
+        reader_kwargs (dict, optional): Keyword arguments for the reader function. Defaults to None.
+        load_args (list, optional): Positional arguments for loading. Defaults to None.
+        load_kwargs (dict, optional): Keyword arguments for loading. Defaults to None.
+        to_lion (bool | Callable, optional): Whether to convert the data to Node instances or a custom parser. Defaults to True.
+
+    Returns:
+        pile: A pile of Node instances.
+
+    Raises:
+        ValueError: If the reader_type is not supported.
+
+    Example usage:
+        >>> nodes = load(input_dir='path/to/text/files', required_exts=['txt'])
+    """
 
     if reader_args is None:
         reader_args = []
@@ -50,11 +76,15 @@ def load(
         load_kwargs = {}
 
     if reader_type == ReaderType.PLAIN:
+        reader_kwargs["dir_"] = input_dir
+        reader_kwargs["ext"] = required_exts
+        reader_kwargs["recursive"] = recursive
+
         return read_funcs[ReaderType.PLAIN](reader, reader_args, reader_kwargs)
 
     if reader_type == ReaderType.LANGCHAIN:
         return read_funcs[ReaderType.LANGCHAIN](
-            reader, reader_args, reader_kwargs, to_datanode
+            reader, reader_args, reader_kwargs, to_lion
         )
 
     elif reader_type == ReaderType.LLAMAINDEX:
@@ -68,12 +98,12 @@ def load(
             reader_kwargs["required_exts"] = required_exts
 
         return read_funcs[ReaderType.LLAMAINDEX](
-            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_datanode
+            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_lion
         )
 
     elif reader_type == ReaderType.SELFDEFINED:
         return read_funcs[ReaderType.SELFDEFINED](
-            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_datanode
+            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_lion
         )
 
     else:
@@ -83,23 +113,56 @@ def load(
 
 
 def _plain_reader(reader, reader_args, reader_kwargs):
+    """
+    Reads data using a plain reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+
+    Returns:
+        pile: A pile of Node instances.
+
+    Raises:
+        ValueError: If the reader is not supported.
+
+    Example usage:
+        >>> nodes = _plain_reader('text_reader', ['path/to/files'], {'ext': 'txt'})
+    """
     try:
         if reader == "text_reader":
             reader = text_reader
-        return reader(reader_args, reader_kwargs)
+        nodes = reader(reader_args, reader_kwargs)
+        return pile(nodes)
     except Exception as e:
         raise ValueError(
             f"Reader {reader} is currently not supported. Error: {e}"
         ) from e
 
 
-def _langchain_reader(reader, reader_args, reader_kwargs, to_datanode: bool | Callable):
+def _langchain_reader(reader, reader_args, reader_kwargs, to_lion: bool | Callable):
+    """
+    Reads data using a Langchain reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> nodes = _langchain_reader('langchain_reader', ['arg1'], {'key': 'value'}, True)
+    """
     nodes = LangchainBridge.langchain_loader(reader, reader_args, reader_kwargs)
-    if isinstance(to_datanode, bool) and to_datanode is True:
-        nodes = [Node.from_langchain(i) for i in nodes]
+    if isinstance(to_lion, bool) and to_lion is True:
+        return pile([Node.from_langchain(i) for i in nodes])
 
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
@@ -109,15 +172,33 @@ def _llama_index_reader(
     reader_kwargs,
     load_args,
     load_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Reads data using a LlamaIndex reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        load_args (list): Positional arguments for loading.
+        load_kwargs (dict): Keyword arguments for loading.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> nodes = _llama_index_reader('llama_reader', ['arg1'], {'key': 'value'}, [], {}, True)
+    """
     nodes = LlamaIndexBridge.llama_index_read_data(
         reader, reader_args, reader_kwargs, load_args, load_kwargs
     )
-    if isinstance(to_datanode, bool) and to_datanode is True:
-        nodes = [Node.from_llama_index(i) for i in nodes]
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    if isinstance(to_lion, bool) and to_lion is True:
+        return pile([Node.from_llama_index(i) for i in nodes])
+
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
@@ -127,8 +208,28 @@ def _self_defined_reader(
     reader_kwargs,
     load_args,
     load_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Reads data using a self-defined reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        load_args (list): Positional arguments for loading.
+        load_kwargs (dict): Keyword arguments for loading.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Raises:
+        ValueError: If the self-defined reader is not valid.
+
+    Example usage:
+        >>> nodes = _self_defined_reader(custom_reader, ['arg1'], {'key': 'value'}, [], {}, custom_parser)
+    """
     try:
         loader = reader(*reader_args, **reader_kwargs)
         nodes = loader.load(*load_args, **load_kwargs)
@@ -137,10 +238,10 @@ def _self_defined_reader(
             f"Self defined reader {reader} is not valid. Error: {e}"
         ) from e
 
-    if isinstance(to_datanode, bool) and to_datanode is True:
+    if isinstance(to_lion, bool) and to_lion is True:
         raise ValueError("Please define a valid parser to Node.")
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes