lionagi 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

lionagi/integrations/config/oai_configs.py

@@ -1,8 +1,10 @@
 # Default configs for the OpenAI API
 
+API_key_schema = ("OPENAI_API_KEY",)
+
 # ChatCompletion
 oai_chat_llmconfig = {
-    "model": "gpt-4-turbo",
+    "model": "gpt-4o",
     "frequency_penalty": 0,
     "max_tokens": None,
     "n": 1,
@@ -11,11 +13,13 @@ oai_chat_llmconfig = {
     "seed": None,
     "stop": None,
     "stream": False,
-    "temperature": 0.7,
+    "temperature": 0.1,
     "top_p": 1,
     "tools": None,
     "tool_choice": "none",
     "user": None,
+    "logprobs": False,
+    "top_logprobs": None,
 }
 
 oai_chat_schema = {
@@ -36,9 +40,16 @@ oai_chat_schema = {
         "tool_choice",
         "user",
         "max_tokens",
+        "logprobs",
+        "top_logprobs",
     ],
     "input_": "messages",
     "config": oai_chat_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 128_000,
+    "interval_tokens": 1_000_000,
+    "interval_requests": 1_000,
+    "interval": 60,
 }
 
 # Finetune
@@ -111,8 +122,26 @@ oai_audio_translations_schema = {
     "config": oai_audio_translations_llmconfig,
 }
 
-# images
+# embeddings
 
+oai_embeddings_llmconfig = {
+    "model": "text-embedding-ada-002",
+    "encoding_format": "float",
+    "user": None,
+    "dimensions": None,
+}
+
+oai_embeddings_schema = {
+    "required": ["model", "encoding_format"],
+    "optional": ["user", "dimensions"],
+    "input_": "input",
+    "config": oai_embeddings_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 8192,
+    "interval_tokens": 1_000_000,
+    "interval_requests": 1_000,
+    "interval": 60,
+}
 
 oai_schema = {
     "chat/completions": oai_chat_schema,
@@ -120,4 +149,6 @@ oai_schema = {
     "audio_speech": oai_audio_speech_schema,
     "audio_transcriptions": oai_audio_transcriptions_schema,
     "audio_translations": oai_audio_translations_schema,
+    "API_key_schema": API_key_schema,
+    "embeddings": oai_embeddings_schema,
 }

lionagi/integrations/config/openrouter_configs.py

@@ -1,5 +1,7 @@
+API_key_schema = ("OPENROUTER_API_KEY",)
+
 openrouter_chat_llmconfig = {
-    "model": "gpt-4-turbo",
+    "model": "gpt-4o",
     "frequency_penalty": 0,
     "max_tokens": None,
     "num": 1,
@@ -8,11 +10,13 @@ openrouter_chat_llmconfig = {
     "seed": None,
     "stop": None,
     "stream": False,
-    "temperature": 0.7,
+    "temperature": 0.1,
     "top_p": 1,
     "tools": None,
     "tool_choice": "none",
     "user": None,
+    "logprobs": False,
+    "top_logprobs": None,
 }
 
 openrouter_chat_schema = {
@@ -33,9 +37,16 @@ openrouter_chat_schema = {
         "tool_choice",
         "user",
         "max_tokens",
+        "logprobs",
+        "top_logprobs",
     ],
     "input_": "messages",
     "config": openrouter_chat_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 128_000,
+    "interval_tokens": 10_000,
+    "interval_requests": 100,
+    "interval": 60,
 }
 
 openrouter_finetune_llmconfig = {
@@ -59,4 +70,5 @@ openrouter_finetune_schema = {
 openrouter_schema = {
     "chat/completions": openrouter_chat_schema,
     "finetune": openrouter_finetune_schema,
+    "API_key_schema": API_key_schema,
 }

lionagi/integrations/loader/load.py

@@ -1,6 +1,7 @@
 from typing import Callable
 
 from lionagi.core.generic import Node
+from lionagi.core.collections import pile
 from ..bridge.langchain_.langchain_bridge import LangchainBridge
 from ..bridge.llamaindex_.llama_index_bridge import LlamaIndexBridge
 
@@ -27,18 +28,43 @@ def text_reader(args, kwargs):
 
 
 def load(
-    reader: str | Callable = "SimpleDirectoryReader",
+    reader: str | Callable = "text_reader",
     input_dir=None,
     input_files=None,
     recursive: bool = False,
     required_exts: list[str] = None,
-    reader_type=ReaderType.LLAMAINDEX,
+    reader_type=ReaderType.PLAIN,
     reader_args=None,
     reader_kwargs=None,
     load_args=None,
     load_kwargs=None,
-    to_datanode: bool | Callable = True,
+    to_lion: bool | Callable = True,
 ):
+    """
+    Loads data using the specified reader and converts it to Node instances.
+
+    Args:
+        reader (str | Callable): The reader function or its name. Defaults to "text_reader".
+        input_dir (str, optional): The directory to read files from. Defaults to None.
+        input_files (list[str], optional): Specific files to read. Defaults to None.
+        recursive (bool, optional): Whether to read files recursively. Defaults to False.
+        required_exts (list[str], optional): List of required file extensions. Defaults to None.
+        reader_type (ReaderType, optional): The type of reader to use. Defaults to ReaderType.PLAIN.
+        reader_args (list, optional): Positional arguments for the reader function. Defaults to None.
+        reader_kwargs (dict, optional): Keyword arguments for the reader function. Defaults to None.
+        load_args (list, optional): Positional arguments for loading. Defaults to None.
+        load_kwargs (dict, optional): Keyword arguments for loading. Defaults to None.
+        to_lion (bool | Callable, optional): Whether to convert the data to Node instances or a custom parser. Defaults to True.
+
+    Returns:
+        pile: A pile of Node instances.
+
+    Raises:
+        ValueError: If the reader_type is not supported.
+
+    Example usage:
+        >>> nodes = load(input_dir='path/to/text/files', required_exts=['txt'])
+    """
 
     if reader_args is None:
         reader_args = []
@@ -50,11 +76,15 @@ def load(
         load_kwargs = {}
 
     if reader_type == ReaderType.PLAIN:
+        reader_kwargs["dir_"] = input_dir
+        reader_kwargs["ext"] = required_exts
+        reader_kwargs["recursive"] = recursive
+
         return read_funcs[ReaderType.PLAIN](reader, reader_args, reader_kwargs)
 
     if reader_type == ReaderType.LANGCHAIN:
         return read_funcs[ReaderType.LANGCHAIN](
-            reader, reader_args, reader_kwargs, to_datanode
+            reader, reader_args, reader_kwargs, to_lion
         )
 
     elif reader_type == ReaderType.LLAMAINDEX:
@@ -68,12 +98,12 @@ def load(
             reader_kwargs["required_exts"] = required_exts
 
         return read_funcs[ReaderType.LLAMAINDEX](
-            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_datanode
+            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_lion
         )
 
     elif reader_type == ReaderType.SELFDEFINED:
         return read_funcs[ReaderType.SELFDEFINED](
-            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_datanode
+            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_lion
         )
 
     else:
@@ -83,23 +113,56 @@ def load(
 
 
 def _plain_reader(reader, reader_args, reader_kwargs):
+    """
+    Reads data using a plain reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+
+    Returns:
+        pile: A pile of Node instances.
+
+    Raises:
+        ValueError: If the reader is not supported.
+
+    Example usage:
+        >>> nodes = _plain_reader('text_reader', ['path/to/files'], {'ext': 'txt'})
+    """
     try:
         if reader == "text_reader":
             reader = text_reader
-        return reader(reader_args, reader_kwargs)
+        nodes = reader(reader_args, reader_kwargs)
+        return pile(nodes)
     except Exception as e:
         raise ValueError(
             f"Reader {reader} is currently not supported. Error: {e}"
         ) from e
 
 
-def _langchain_reader(reader, reader_args, reader_kwargs, to_datanode: bool | Callable):
+def _langchain_reader(reader, reader_args, reader_kwargs, to_lion: bool | Callable):
+    """
+    Reads data using a Langchain reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> nodes = _langchain_reader('langchain_reader', ['arg1'], {'key': 'value'}, True)
+    """
     nodes = LangchainBridge.langchain_loader(reader, reader_args, reader_kwargs)
-    if isinstance(to_datanode, bool) and to_datanode is True:
-        nodes = [Node.from_langchain(i) for i in nodes]
+    if isinstance(to_lion, bool) and to_lion is True:
+        return pile([Node.from_langchain(i) for i in nodes])
 
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
@@ -109,15 +172,33 @@ def _llama_index_reader(
     reader_kwargs,
     load_args,
     load_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Reads data using a LlamaIndex reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        load_args (list): Positional arguments for loading.
+        load_kwargs (dict): Keyword arguments for loading.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> nodes = _llama_index_reader('llama_reader', ['arg1'], {'key': 'value'}, [], {}, True)
+    """
     nodes = LlamaIndexBridge.llama_index_read_data(
         reader, reader_args, reader_kwargs, load_args, load_kwargs
     )
-    if isinstance(to_datanode, bool) and to_datanode is True:
-        nodes = [Node.from_llama_index(i) for i in nodes]
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    if isinstance(to_lion, bool) and to_lion is True:
+        return pile([Node.from_llama_index(i) for i in nodes])
+
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
@@ -127,8 +208,28 @@ def _self_defined_reader(
     reader_kwargs,
     load_args,
     load_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Reads data using a self-defined reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        load_args (list): Positional arguments for loading.
+        load_kwargs (dict): Keyword arguments for loading.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Raises:
+        ValueError: If the self-defined reader is not valid.
+
+    Example usage:
+        >>> nodes = _self_defined_reader(custom_reader, ['arg1'], {'key': 'value'}, [], {}, custom_parser)
+    """
     try:
         loader = reader(*reader_args, **reader_kwargs)
         nodes = loader.load(*load_args, **load_kwargs)
@@ -137,10 +238,10 @@ def _self_defined_reader(
             f"Self defined reader {reader} is not valid. Error: {e}"
         ) from e
 
-    if isinstance(to_datanode, bool) and to_datanode is True:
+    if isinstance(to_lion, bool) and to_lion is True:
         raise ValueError("Please define a valid parser to Node.")
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 

lionagi/integrations/loader/load_util.py

@@ -1,10 +1,10 @@
 # use utils and schema
-import math
 from enum import Enum
 from pathlib import Path
 from typing import List, Union, Dict, Any, Tuple
 
 from lionagi.libs import convert, func_call
+from lionagi.libs.ln_tokenize import TokenizeUtil
 from lionagi.core.generic import Node
 
 
@@ -57,7 +57,7 @@ def dir_to_path(
 
 
 def dir_to_nodes(
-    dir: str,
+    dir_: str,
     ext: Union[List[str], str],
     recursive: bool = False,
     flatten: bool = True,
@@ -84,82 +84,11 @@ def dir_to_nodes(
         # converting them into Node objects.
     """
 
-    path_list = dir_to_path(dir, ext, recursive, flatten)
+    path_list = dir_to_path(dir=dir_, ext=ext, recursive=recursive, flatten=flatten)
     files_info = func_call.lcall(path_list, read_text, clean=clean_text)
     return func_call.lcall(files_info, lambda x: Node(content=x[0], metadata=x[1]))
 
 
-def chunk_text(
-    input: str, chunk_size: int, overlap: float, threshold: int
-) -> List[Union[str, None]]:
-    """
-    Chunks the input text into smaller parts, with optional overlap and threshold for final chunk.
-
-    Parameters:
-        input (str): The input text to chunk.
-
-        chunk_size (int): The size of each chunk.
-
-        overlap (float): The amount of overlap between chunks.
-
-        threshold (int): The minimum size of the final chunk.
-
-    Returns:
-        List[Union[str, None]]: A list of text chunks.
-
-    Raises:
-        ValueError: If an error occurs during chunking.
-    """
-
-    def _chunk_n1():
-        return [input]
-
-    def _chunk_n2():
-        chunks = []
-        chunks.append(input[: chunk_size + overlap_size])
-
-        if len(input) - chunk_size > threshold:
-            chunks.append(input[chunk_size - overlap_size :])
-        else:
-            return _chunk_n1()
-
-        return chunks
-
-    def _chunk_n3():
-        chunks = []
-        chunks.append(input[: chunk_size + overlap_size])
-        for i in range(1, n_chunks - 1):
-            start_idx = chunk_size * i - overlap_size
-            end_idx = chunk_size * (i + 1) + overlap_size
-            chunks.append(input[start_idx:end_idx])
-
-        if len(input) - chunk_size * (n_chunks - 1) > threshold:
-            chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size :])
-        else:
-            chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size :]
-
-        return chunks
-
-    try:
-        if not isinstance(input, str):
-            input = convert.to_str(input)
-
-        n_chunks = math.ceil(len(input) / chunk_size)
-        overlap_size = int(overlap / 2)
-
-        if n_chunks == 1:
-            return _chunk_n1()
-
-        elif n_chunks == 2:
-            return _chunk_n2()
-
-        elif n_chunks > 2:
-            return _chunk_n3()
-
-    except Exception as e:
-        raise ValueError(f"An error occurred while chunking the text. {e}")
-
-
 def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
     """
     Reads text from a file and optionally cleans it, returning the content and metadata.
@@ -202,9 +131,9 @@ def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
             content = f.read()
             if clean:
                 # Define characters to replace and their replacements
-                replacements = {"\\": " ", "\n": " ", "\t": " ", "  ": " ", "'": " "}
+                replacements = {"\\": "", "\n\n": "\n"}
                 for old, new in replacements.items():
-                    content = content.replace(old, new)
+                    content = content.replace(old, new).strip()
             metadata = _get_metadata()
             return content, metadata
     except Exception as e:
@@ -223,7 +152,7 @@ def _file_to_chunks(
             "chunk_overlap": overlap,
             "chunk_threshold": threshold,
         }
-        chunks = chunk_text(
+        chunks = TokenizeUtil.chunk_by_chars(
             input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold
         )
         logs = []

lionagi/integrations/provider/_mapping.py

@@ -0,0 +1,46 @@
+from .oai import OpenAIService
+from .openrouter import OpenRouterService
+from .ollama import OllamaService
+from .transformers import TransformersService
+from .litellm import LiteLLMService
+from .mlx_service import MLXService
+from lionagi.integrations.config.oai_configs import oai_schema
+from lionagi.integrations.config.openrouter_configs import openrouter_schema
+
+SERVICE_PROVIDERS_MAPPING = {
+    "openai": {
+        "service": OpenAIService,
+        "schema": oai_schema,
+        "default_model": "gpt-3.5-turbo",
+    },
+    "openrouter": {
+        "service": OpenRouterService,
+        "schema": openrouter_schema,
+        "default_model": "gpt-3.5-turbo",
+    },
+    "litellm": {
+        "service": LiteLLMService,
+        "schema": oai_schema,
+        "default_model": "gpt-3.5-turbo",
+    },
+    "ollama": {
+        "service": OllamaService,
+        "schema": {"model": "llama3"},
+        "default_model": "llama3",
+    },
+    "transformers": {
+        "service": TransformersService,
+        "schema": {"model": "gpt2"},
+        "default_model": "gpt2",
+    },
+    "mlx": {
+        "service": MLXService,
+        "schema": {"model": "mlx-community/OLMo-7B-hf-4bit-mlx"},
+        "default_model": "mlx-community/OLMo-7B-hf-4bit-mlx",
+    },
+}
+
+# TODO
+# "Ollama": OllamaService,
+# "Transformers": TransformersService,
+# "MLX": MLXService,

lionagi/integrations/provider/litellm.py

@@ -31,12 +31,13 @@ class LiteLLMService(BaseService):
         self.acompletion = acompletion
         self.model = model
         self.kwargs = kwargs
+        self.allowed_kwargs = allowed_kwargs
 
     async def serve_chat(self, messages, **kwargs):
         payload = {"messages": messages}
         config = {}
         for k, v in kwargs.items():
-            if k in allowed_kwargs:
+            if k in self.allowed_kwargs:
                 config[k] = v
 
         kwargs = {**self.kwargs, **config}

lionagi/integrations/provider/mlx_service.py

@@ -1,13 +1,18 @@
+import re
 from lionagi.libs.sys_util import SysUtil
 import lionagi.libs.ln_convert as convert
 from lionagi.libs.ln_api import BaseService
 from lionagi.integrations.config.mlx_configs import model
 
 
-class MlXService(BaseService):
+class MLXService(BaseService):
     def __init__(self, model=model, **kwargs):
 
         SysUtil.check_import("mlx_lm")
+        SysUtil.check_import("ipywidgets")
+
+        if model is not None and "olmo" in str(model).lower():
+            SysUtil.check_import("olmo", pip_name="ai2-olmo")
 
         from mlx_lm import load, generate
 
@@ -19,16 +24,13 @@ class MlXService(BaseService):
         self.model = model_
         self.tokenizer = tokenizer
         self.generate = generate
+        self.allowed_kwargs = []
 
     async def serve_chat(self, messages, **kwargs):
         if "verbose" not in kwargs.keys():
-            verbose = True
+            verbose = False
 
-        prompts = [
-            convert.to_dict(msg["content"])["instruction"]
-            for msg in messages
-            if msg["role"] == "user"
-        ]
+        prompts = [msg["content"] for msg in messages if msg["role"] == "user"]
 
         payload = {"messages": messages}
 
@@ -39,8 +41,13 @@ class MlXService(BaseService):
                 prompt=f"{prompts[-1]} \nOutput: ",
                 verbose=verbose,
             )
-            completion = {"model": self.model_name, "choices": [{"message": response}]}
-
+            if "```" in response:
+                regex = re.compile(r"```[\s\S]*?```")
+                matches = regex.findall(response)
+                msg = matches[0].strip("```")
+                completion = {"choices": [{"message": {"content": msg}}]}
+            else:
+                completion = {"choices": [{"message": {"content": response}}]}
             return payload, completion
         except Exception as e:
             self.status_tracker.num_tasks_failed += 1