lionagi 0.0.312__py3-none-any.whl → 0.2.1__py3-none-any.whl

lionagi/integrations/config/oai_configs.py

@@ -1,8 +1,10 @@
 # Default configs for the OpenAI API
 
+API_key_schema = ("OPENAI_API_KEY",)
+
 # ChatCompletion
 oai_chat_llmconfig = {
-    "model": "gpt-4-turbo-preview",
+    "model": "gpt-4o",
     "frequency_penalty": 0,
     "max_tokens": None,
     "n": 1,
@@ -11,11 +13,13 @@ oai_chat_llmconfig = {
     "seed": None,
     "stop": None,
     "stream": False,
-    "temperature": 0.7,
+    "temperature": 0.1,
     "top_p": 1,
     "tools": None,
     "tool_choice": "none",
     "user": None,
+    "logprobs": False,
+    "top_logprobs": None,
 }
 
 oai_chat_schema = {
@@ -36,9 +40,16 @@ oai_chat_schema = {
         "tool_choice",
         "user",
         "max_tokens",
+        "logprobs",
+        "top_logprobs",
     ],
     "input_": "messages",
     "config": oai_chat_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 128_000,
+    "interval_tokens": 1_000_000,
+    "interval_requests": 1_000,
+    "interval": 60,
 }
 
 # Finetune
@@ -79,7 +90,7 @@ oai_audio_speech_schema = {
 oai_audio_transcriptions_llmconfig = {
     "model": "whisper-1",
     "language": None,
-    "prompt": None,
+    "format_prompt": None,
     "response_format": "json",
     "temperature": 0,
 }
@@ -88,7 +99,7 @@ oai_audio_transcriptions_schema = {
     "optional": [
         "response_format",
         "language",
-        "prompt",
+        "format_prompt",
         "response_format",
         "temperature",
     ],
@@ -99,20 +110,38 @@ oai_audio_transcriptions_schema = {
 # Audio ------------    translations
 oai_audio_translations_llmconfig = {
     "model": "whisper-1",
-    "prompt": None,
+    "format_prompt": None,
     "response_format": "json",
     "temperature": 0,
 }
 
 oai_audio_translations_schema = {
     "required": ["model"],
-    "optional": ["response_format", "speed", "prompt", "temperature"],
+    "optional": ["response_format", "speed", "format_prompt", "temperature"],
     "input_": "file",
     "config": oai_audio_translations_llmconfig,
 }
 
-# images
+# embeddings
 
+oai_embeddings_llmconfig = {
+    "model": "text-embedding-ada-002",
+    "encoding_format": "float",
+    "user": None,
+    "dimensions": None,
+}
+
+oai_embeddings_schema = {
+    "required": ["model", "encoding_format"],
+    "optional": ["user", "dimensions"],
+    "input_": "input",
+    "config": oai_embeddings_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 8192,
+    "interval_tokens": 1_000_000,
+    "interval_requests": 1_000,
+    "interval": 60,
+}
 
 oai_schema = {
     "chat/completions": oai_chat_schema,
@@ -120,4 +149,6 @@ oai_schema = {
     "audio_speech": oai_audio_speech_schema,
     "audio_transcriptions": oai_audio_transcriptions_schema,
     "audio_translations": oai_audio_translations_schema,
+    "API_key_schema": API_key_schema,
+    "embeddings": oai_embeddings_schema,
 }

lionagi/integrations/config/ollama_configs.py

@@ -1 +1 @@
-model = "llama2"
+model = "llama3"

lionagi/integrations/config/openrouter_configs.py

@@ -1,5 +1,7 @@
+API_key_schema = ("OPENROUTER_API_KEY",)
+
 openrouter_chat_llmconfig = {
-    "model": "gpt-4-turbo-preview",
+    "model": "gpt-4o",
     "frequency_penalty": 0,
     "max_tokens": None,
     "num": 1,
@@ -8,11 +10,13 @@ openrouter_chat_llmconfig = {
     "seed": None,
     "stop": None,
     "stream": False,
-    "temperature": 0.7,
+    "temperature": 0.1,
     "top_p": 1,
     "tools": None,
     "tool_choice": "none",
     "user": None,
+    "logprobs": False,
+    "top_logprobs": None,
 }
 
 openrouter_chat_schema = {
@@ -33,9 +37,16 @@ openrouter_chat_schema = {
         "tool_choice",
         "user",
         "max_tokens",
+        "logprobs",
+        "top_logprobs",
     ],
     "input_": "messages",
     "config": openrouter_chat_llmconfig,
+    "token_encoding_name": "cl100k_base",
+    "token_limit": 128_000,
+    "interval_tokens": 10_000,
+    "interval_requests": 100,
+    "interval": 60,
 }
 
 openrouter_finetune_llmconfig = {
@@ -59,4 +70,5 @@ openrouter_finetune_schema = {
 openrouter_schema = {
     "chat/completions": openrouter_chat_schema,
     "finetune": openrouter_finetune_schema,
+    "API_key_schema": API_key_schema,
 }

lionagi/integrations/loader/load.py

@@ -0,0 +1,253 @@
+from typing import Callable
+
+from lionagi.core.generic import Node
+from lionagi.core.collections import pile
+from ..bridge.langchain_.langchain_bridge import LangchainBridge
+from ..bridge.llamaindex_.llama_index_bridge import LlamaIndexBridge
+
+from .load_util import dir_to_nodes, ReaderType, _datanode_parser
+
+
+def text_reader(args, kwargs):
+    """
+    Reads text files from a directory and converts them to Node instances.
+
+    Args:
+        args: Positional arguments for the dir_to_nodes function.
+        kwargs: Keyword arguments for the dir_to_nodes function.
+
+    Returns:
+        A list of Node instances.
+
+    Example usage:
+        >>> args = ['path/to/text/files']
+        >>> kwargs = {'file_extension': 'txt'}
+        >>> nodes = text_reader(args, kwargs)
+    """
+    return dir_to_nodes(*args, **kwargs)
+
+
+def load(
+    reader: str | Callable = "text_reader",
+    input_dir=None,
+    input_files=None,
+    recursive: bool = False,
+    required_exts: list[str] = None,
+    reader_type=ReaderType.PLAIN,
+    reader_args=None,
+    reader_kwargs=None,
+    load_args=None,
+    load_kwargs=None,
+    to_lion: bool | Callable = True,
+):
+    """
+    Loads data using the specified reader and converts it to Node instances.
+
+    Args:
+        reader (str | Callable): The reader function or its name. Defaults to "text_reader".
+        input_dir (str, optional): The directory to read files from. Defaults to None.
+        input_files (list[str], optional): Specific files to read. Defaults to None.
+        recursive (bool, optional): Whether to read files recursively. Defaults to False.
+        required_exts (list[str], optional): List of required file extensions. Defaults to None.
+        reader_type (ReaderType, optional): The type of reader to use. Defaults to ReaderType.PLAIN.
+        reader_args (list, optional): Positional arguments for the reader function. Defaults to None.
+        reader_kwargs (dict, optional): Keyword arguments for the reader function. Defaults to None.
+        load_args (list, optional): Positional arguments for loading. Defaults to None.
+        load_kwargs (dict, optional): Keyword arguments for loading. Defaults to None.
+        to_lion (bool | Callable, optional): Whether to convert the data to Node instances or a custom parser. Defaults to True.
+
+    Returns:
+        pile: A pile of Node instances.
+
+    Raises:
+        ValueError: If the reader_type is not supported.
+
+    Example usage:
+        >>> nodes = load(input_dir='path/to/text/files', required_exts=['txt'])
+    """
+
+    if reader_args is None:
+        reader_args = []
+    if reader_kwargs is None:
+        reader_kwargs = {}
+    if load_args is None:
+        load_args = []
+    if load_kwargs is None:
+        load_kwargs = {}
+
+    if reader_type == ReaderType.PLAIN:
+        reader_kwargs["dir_"] = input_dir
+        reader_kwargs["ext"] = required_exts
+        reader_kwargs["recursive"] = recursive
+
+        return read_funcs[ReaderType.PLAIN](reader, reader_args, reader_kwargs)
+
+    if reader_type == ReaderType.LANGCHAIN:
+        return read_funcs[ReaderType.LANGCHAIN](
+            reader, reader_args, reader_kwargs, to_lion
+        )
+
+    elif reader_type == ReaderType.LLAMAINDEX:
+        if input_dir is not None:
+            reader_kwargs["input_dir"] = input_dir
+        if input_files is not None:
+            reader_kwargs["input_files"] = input_files
+        if recursive:
+            reader_kwargs["recursive"] = True
+        if required_exts is not None:
+            reader_kwargs["required_exts"] = required_exts
+
+        return read_funcs[ReaderType.LLAMAINDEX](
+            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_lion
+        )
+
+    elif reader_type == ReaderType.SELFDEFINED:
+        return read_funcs[ReaderType.SELFDEFINED](
+            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_lion
+        )
+
+    else:
+        raise ValueError(
+            f"{reader_type} is not supported. Please choose from {list(ReaderType)}"
+        )
+
+
+def _plain_reader(reader, reader_args, reader_kwargs):
+    """
+    Reads data using a plain reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+
+    Returns:
+        pile: A pile of Node instances.
+
+    Raises:
+        ValueError: If the reader is not supported.
+
+    Example usage:
+        >>> nodes = _plain_reader('text_reader', ['path/to/files'], {'ext': 'txt'})
+    """
+    try:
+        if reader == "text_reader":
+            reader = text_reader
+        nodes = reader(reader_args, reader_kwargs)
+        return pile(nodes)
+    except Exception as e:
+        raise ValueError(
+            f"Reader {reader} is currently not supported. Error: {e}"
+        ) from e
+
+
+def _langchain_reader(reader, reader_args, reader_kwargs, to_lion: bool | Callable):
+    """
+    Reads data using a Langchain reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> nodes = _langchain_reader('langchain_reader', ['arg1'], {'key': 'value'}, True)
+    """
+    nodes = LangchainBridge.langchain_loader(reader, reader_args, reader_kwargs)
+    if isinstance(to_lion, bool) and to_lion is True:
+        return pile([Node.from_langchain(i) for i in nodes])
+
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
+    return nodes
+
+
+def _llama_index_reader(
+    reader,
+    reader_args,
+    reader_kwargs,
+    load_args,
+    load_kwargs,
+    to_lion: bool | Callable,
+):
+    """
+    Reads data using a LlamaIndex reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        load_args (list): Positional arguments for loading.
+        load_kwargs (dict): Keyword arguments for loading.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> nodes = _llama_index_reader('llama_reader', ['arg1'], {'key': 'value'}, [], {}, True)
+    """
+    nodes = LlamaIndexBridge.llama_index_read_data(
+        reader, reader_args, reader_kwargs, load_args, load_kwargs
+    )
+    if isinstance(to_lion, bool) and to_lion is True:
+        return pile([Node.from_llama_index(i) for i in nodes])
+
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
+    return nodes
+
+
+def _self_defined_reader(
+    reader,
+    reader_args,
+    reader_kwargs,
+    load_args,
+    load_kwargs,
+    to_lion: bool | Callable,
+):
+    """
+    Reads data using a self-defined reader.
+
+    Args:
+        reader (str | Callable): The reader function or its name.
+        reader_args (list): Positional arguments for the reader function.
+        reader_kwargs (dict): Keyword arguments for the reader function.
+        load_args (list): Positional arguments for loading.
+        load_kwargs (dict): Keyword arguments for loading.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of Node instances or custom parsed nodes.
+
+    Raises:
+        ValueError: If the self-defined reader is not valid.
+
+    Example usage:
+        >>> nodes = _self_defined_reader(custom_reader, ['arg1'], {'key': 'value'}, [], {}, custom_parser)
+    """
+    try:
+        loader = reader(*reader_args, **reader_kwargs)
+        nodes = loader.load(*load_args, **load_kwargs)
+    except Exception as e:
+        raise ValueError(
+            f"Self defined reader {reader} is not valid. Error: {e}"
+        ) from e
+
+    if isinstance(to_lion, bool) and to_lion is True:
+        raise ValueError("Please define a valid parser to Node.")
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
+    return nodes
+
+
+read_funcs = {
+    ReaderType.PLAIN: _plain_reader,
+    ReaderType.LANGCHAIN: _langchain_reader,
+    ReaderType.LLAMAINDEX: _llama_index_reader,
+    ReaderType.SELFDEFINED: _self_defined_reader,
+}

lionagi/integrations/loader/load_util.py

@@ -0,0 +1,195 @@
+# use utils and schema
+from enum import Enum
+from pathlib import Path
+from typing import List, Union, Dict, Any, Tuple
+
+from lionagi.libs import convert, func_call
+from lionagi.libs.ln_tokenize import TokenizeUtil
+from lionagi.core.generic import Node
+
+
+class ReaderType(str, Enum):
+    PLAIN = "plain"
+    LANGCHAIN = "langchain"
+    LLAMAINDEX = "llama_index"
+    SELFDEFINED = "self_defined"
+
+
+class ChunkerType(str, Enum):
+    PLAIN = "plain"  # default
+    LANGCHAIN = "langchain"  # using langchain functions
+    LLAMAINDEX = "llama_index"  # using llamaindex functions
+    SELFDEFINED = "self_defined"  # create custom functions
+
+
+def dir_to_path(
+    dir: str, ext: str, recursive: bool = False, flatten: bool = True
+) -> List[Path]:
+    """
+    Generates a list of file paths from a directory with the given file extension.
+
+    Parameters:
+        dir (str): The directory to search for files.
+
+        ext (str): The file extension to filter by.
+
+        recursive (bool): Whether to search subdirectories recursively. Defaults to False.
+
+        flatten (bool): Whether to flatten the list. Defaults to True.
+
+    Returns:
+        List[Path]: A list of Paths to the files.
+
+    Raises:
+        ValueError: If the directory or extension is invalid.
+    """
+
+    def _dir_to_path(ext):
+        tem = "**/*" if recursive else "*"
+        return list(Path(dir).glob(tem + ext))
+
+    try:
+        return convert.to_list(
+            func_call.lcall(ext, _dir_to_path, flatten=True), flatten=flatten
+        )
+    except:
+        raise ValueError("Invalid directory or extension, please check the path")
+
+
+def dir_to_nodes(
+    dir_: str,
+    ext: Union[List[str], str],
+    recursive: bool = False,
+    flatten: bool = True,
+    clean_text: bool = True,
+) -> List[Node]:
+    """
+    Converts directory contents into Node objects based on specified file extensions.
+
+    This function first retrieves a list of file paths from the specified directory, matching the given file extension. It then reads the content of these files, optionally cleaning the text, and converts each file's content into a Node object.
+
+    Parameters:
+        dir (str): The directory path from which to read files.
+        ext: The file extension(s) to include. Can be a single string or a list/tuple of strings.
+        recursive (bool, optional): If True, the function searches for files recursively in subdirectories. Defaults to False.
+        flatten (bool, optional): If True, flattens the directory structure in the returned paths. Defaults to True.
+        clean_text (bool, optional): If True, cleans the text read from files. Defaults to True.
+
+    Returns:
+        list: A list of Node objects created from the files in the specified directory.
+
+    Example:
+        nodes = dir_to_nodes("/path/to/dir", ".txt", recursive=True)
+        # This would read all .txt files in /path/to/dir and its subdirectories,
+        # converting them into Node objects.
+    """
+
+    path_list = dir_to_path(dir=dir_, ext=ext, recursive=recursive, flatten=flatten)
+    files_info = func_call.lcall(path_list, read_text, clean=clean_text)
+    return func_call.lcall(files_info, lambda x: Node(content=x[0], metadata=x[1]))
+
+
+def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
+    """
+    Reads text from a file and optionally cleans it, returning the content and metadata.
+
+    Parameters:
+        filepath (str): The path to the file to read.
+
+        clean (bool): Whether to clean the text by replacing certain characters. Defaults to True.
+
+    Returns:
+        Tuple[str, dict]: A tuple containing the content and metadata of the file.
+
+    Raises:
+        FileNotFoundError: If the file cannot be found.
+
+        PermissionError: If there are permissions issues.
+
+        OSError: For other OS-related errors.
+    """
+
+    def _get_metadata():
+        import os
+        from datetime import datetime
+
+        file = filepath
+        size = os.path.getsize(filepath)
+        creation_date = datetime.fromtimestamp(os.path.getctime(filepath)).date()
+        modified_date = datetime.fromtimestamp(os.path.getmtime(filepath)).date()
+        last_accessed_date = datetime.fromtimestamp(os.path.getatime(filepath)).date()
+        return {
+            "file": convert.to_str(file),
+            "size": size,
+            "creation_date": str(creation_date),
+            "modified_date": str(modified_date),
+            "last_accessed_date": str(last_accessed_date),
+        }
+
+    try:
+        with open(filepath, "r") as f:
+            content = f.read()
+            if clean:
+                # Define characters to replace and their replacements
+                replacements = {"\\": "", "\n\n": "\n"}
+                for old, new in replacements.items():
+                    content = content.replace(old, new).strip()
+            metadata = _get_metadata()
+            return content, metadata
+    except Exception as e:
+        raise e
+
+
+def _file_to_chunks(
+    input: Dict[str, Any],
+    field: str = "content",
+    chunk_size: int = 1500,
+    overlap: float = 0.1,
+    threshold: int = 200,
+) -> List[Dict[str, Any]]:
+    try:
+        out = {key: value for key, value in input.items() if key != field} | {
+            "chunk_overlap": overlap,
+            "chunk_threshold": threshold,
+        }
+        chunks = TokenizeUtil.chunk_by_chars(
+            input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold
+        )
+        logs = []
+        for i, chunk in enumerate(chunks):
+            chunk_dict = out | {
+                "file_chunks": len(chunks),
+                "chunk_id": i + 1,
+                "chunk_size": len(chunk),
+                f"chunk_{field}": chunk,
+            }
+            logs.append(chunk_dict)
+
+        return logs
+
+    except Exception as e:
+        raise ValueError(f"An error occurred while chunking the file. {e}") from e
+
+
+# needs doing TODO
+def file_to_chunks(
+    input,
+    #    project='project',
+    #    output_dir='data/logs/sources/',
+    chunk_func=_file_to_chunks,
+    **kwargs,
+):
+    #    out_to_csv=False,
+    #    filename=None,
+    #    verbose=True,
+    #    timestamp=True,
+    #    logger=None,
+    return convert.to_list(func_call.lcall(input, chunk_func, **kwargs), flatten=True)
+
+
+def _datanode_parser(nodes, parser):
+
+    try:
+        return parser(nodes)
+    except Exception as e:
+        raise ValueError(f"Node parser {parser} failed. Error:{e}") from e

lionagi/integrations/provider/_mapping.py

@@ -0,0 +1,46 @@
+from .oai import OpenAIService
+from .openrouter import OpenRouterService
+from .ollama import OllamaService
+from .transformers import TransformersService
+from .litellm import LiteLLMService
+from .mlx_service import MLXService
+from lionagi.integrations.config.oai_configs import oai_schema
+from lionagi.integrations.config.openrouter_configs import openrouter_schema
+
+SERVICE_PROVIDERS_MAPPING = {
+    "openai": {
+        "service": OpenAIService,
+        "schema": oai_schema,
+        "default_model": "gpt-3.5-turbo",
+    },
+    "openrouter": {
+        "service": OpenRouterService,
+        "schema": openrouter_schema,
+        "default_model": "gpt-3.5-turbo",
+    },
+    "litellm": {
+        "service": LiteLLMService,
+        "schema": oai_schema,
+        "default_model": "gpt-3.5-turbo",
+    },
+    "ollama": {
+        "service": OllamaService,
+        "schema": {"model": "llama3"},
+        "default_model": "llama3",
+    },
+    "transformers": {
+        "service": TransformersService,
+        "schema": {"model": "gpt2"},
+        "default_model": "gpt2",
+    },
+    "mlx": {
+        "service": MLXService,
+        "schema": {"model": "mlx-community/OLMo-7B-hf-4bit-mlx"},
+        "default_model": "mlx-community/OLMo-7B-hf-4bit-mlx",
+    },
+}
+
+# TODO
+# "Ollama": OllamaService,
+# "Transformers": TransformersService,
+# "MLX": MLXService,

lionagi/integrations/provider/litellm.py

@@ -31,12 +31,13 @@ class LiteLLMService(BaseService):
         self.acompletion = acompletion
         self.model = model
         self.kwargs = kwargs
+        self.allowed_kwargs = allowed_kwargs
 
     async def serve_chat(self, messages, **kwargs):
         payload = {"messages": messages}
         config = {}
         for k, v in kwargs.items():
-            if k in allowed_kwargs:
+            if k in self.allowed_kwargs:
                 config[k] = v
 
         kwargs = {**self.kwargs, **config}