lionagi 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

lionagi/integrations/bridge/llamaindex_/reader.py

@@ -1,201 +0,0 @@
-from typing import Any
-
-from lionagi.libs.sys_util import SysUtil
-
-
-def get_llama_index_reader(reader: Any | str = None) -> Any:
-    """
-    Retrieves a llama index reader object based on the specified reader name or class.
-
-    This function checks if the specified reader is a recognized type or name and returns the appropriate
-    llama index reader class. If a string is provided, it attempts to match it with known reader names and import
-    the corresponding reader class dynamically. If a reader class is provided, it validates that the class is a
-    subclass of BasePydanticReader.
-
-    Args:
-      reader (Union[Any, str], optional): The reader identifier, which can be a reader class, a string alias
-              for a reader class, or None. If None, returns the SimpleDirectoryReader class.
-
-    Returns:
-      Any: The llama index reader class corresponding to the specified reader.
-
-    Raises:
-      TypeError: If the reader is neither a string nor a subclass of BasePydanticReader.
-      ValueError: If the specified reader string does not correspond to a known reader.
-      AttributeError: If there is an issue importing the specified reader.
-    """
-
-    SysUtil.check_import("llama_index", pip_name="llama-index")
-    from llama_index.core import SimpleDirectoryReader
-    from llama_index.core.readers.base import BasePydanticReader
-
-    if reader in [
-        "SimpleDirectoryReader",
-        SimpleDirectoryReader,
-        "simple-directory-reader",
-        "simple_directory_reader",
-        "simple",
-        "simple_reader",
-        "simple-reader",
-    ]:
-        return SimpleDirectoryReader
-
-    if not isinstance(reader, str) and not issubclass(
-        reader, BasePydanticReader
-    ):
-        raise TypeError(f"reader must be a string or BasePydanticReader.")
-
-    if isinstance(reader, str):
-        package_name, pip_name = parse_reader_name(reader)
-        if package_name == "" and pip_name == "":
-            raise ValueError(
-                f"{reader} is not found. Please directly input llama-index reader class "
-                f"or check llama-index documentation for supported readers."
-            )
-
-        try:
-            SysUtil.check_import(package_name, pip_name=pip_name)
-            reader = getattr(SysUtil.import_module(package_name), reader)
-            return reader
-
-        except Exception as e:
-            raise AttributeError(
-                f"Failed to import/download {reader}, "
-                f"please check llama-index documentation to download it "
-                f"manually and input the reader object: {e}"
-            )
-
-    elif issubclass(reader, BasePydanticReader):
-        return reader
-
-
-def parse_reader_name(reader_str):
-    """
-    Parses a reader name string into a package and pip name.
-
-    Given a reader name string, this function maps it to the corresponding package name and pip name
-    to facilitate dynamic import and installation if necessary.
-
-    Args:
-            reader_str (str): The name of the reader as a string.
-
-    Returns:
-            Tuple[str, str]: A tuple containing the package name and the pip name corresponding to the reader.
-    """
-
-    package_name = ""
-    pip_name = ""
-
-    if reader_str == "PsychicReader":
-        package_name = "llama_index.readers.psychic"
-        pip_name = "llama-index-readers-psychic"
-    elif reader_str == "DeepLakeReader":
-        package_name = "llama_index.readers.deeplake"
-        pip_name = "llama-index-readers-deeplake"
-    elif reader_str == "QdrantReader":
-        package_name = "llama_index.readers.qdrant"
-        pip_name = "llama-index-readers-qdrant"
-    elif reader_str == "DiscordReader":
-        package_name = "llama_index.readers.discord"
-        pip_name = "llama-index-readers-discord"
-    elif reader_str == "SimpleMongoReader":
-        package_name = "llama_index.readers.mongodb"
-        pip_name = "llama-index-readers-mongodb"
-    elif reader_str == "ChromaReader":
-        package_name = "llama_index.readers.chroma"
-        pip_name = "llama-index-readers-chroma"
-    elif reader_str == "MyScaleReader":
-        package_name = "llama_index.readers.myscale"
-        pip_name = "llama-index-readers-myscale"
-    elif reader_str == "FaissReader":
-        package_name = "llama_index.readers.faiss"
-        pip_name = "llama-index-readers-faiss"
-    elif reader_str == "ObsidianReader":
-        package_name = "llama_index.readers.obsidian"
-        pip_name = "llama-index-readers-obsidian"
-    elif reader_str == "SlackReader":
-        package_name = "llama_index.readers.slack"
-        pip_name = "llama-index-readers-slack"
-    elif reader_str == "SimpleWebPageReader":
-        package_name = "llama_index.readers.web"
-        pip_name = "llama-index-readers-web"
-    elif reader_str == "PineconeReader":
-        package_name = "llama_index.readers.pinecone"
-        pip_name = "llama-index-readers-pinecone"
-    elif reader_str == "PathwayReader":
-        package_name = "llama_index.readers.pathway"
-        pip_name = "llama-index-readers-pathway"
-    elif reader_str == "MboxReader":
-        package_name = "llama_index.readers.mbox"
-        pip_name = "llama-index-readers-mbox"
-    elif reader_str == "MilvusReader":
-        package_name = "llama_index.readers.milvus"
-        pip_name = "llama-index-readers-milvus"
-    elif reader_str == "NotionPageReader":
-        package_name = "llama_index.readers.notion"
-        pip_name = "llama-index-readers-notion"
-    elif reader_str == "GithubRepositoryReader":
-        package_name = "llama_index.readers.github"
-        pip_name = "llama-index-readers-github"
-    elif reader_str == "GoogleDocsReader":
-        package_name = "llama_index.readers.google"
-        pip_name = "llama-index-readers-google"
-    elif reader_str == "DatabaseReader":
-        package_name = "llama_index.readers.database"
-        pip_name = "llama-index-readers-database"
-    elif reader_str == "TwitterTweetReader":
-        package_name = "llama_index.readers.twitter"
-        pip_name = "llama-index-readers-twitter"
-    elif reader_str == "WeaviateReader":
-        package_name = "llama_index.readers.weaviate"
-        pip_name = "llama-index-readers-weaviate"
-    elif reader_str == "PandasAIReader":
-        package_name = "llama_index.readers.pandas_ai"
-        pip_name = "llama-index-readers-pandas-ai"
-    elif reader_str == "IntercomReader":
-        package_name = "llama_index.readers.intercom"
-        pip_name = "llama-index-readers-intercom"
-
-    return package_name, pip_name
-
-
-def llama_index_read_data(
-    reader=None,
-    reader_args=None,
-    reader_kwargs=None,
-    loader_args=None,
-    loader_kwargs=None,
-):
-    """
-    Reads data using a specified llama index reader and its arguments.
-
-    This function initializes a llama index reader with the given arguments and keyword arguments,
-    then loads data using the reader's `load_data` method with the provided loader arguments and keyword arguments.
-
-    Args:
-            reader (Union[None, str, Any], optional): The reader to use. This can be a class, a string identifier,
-                    or None. If None, a default reader is used.
-            reader_args (List[Any], optional): Positional arguments to initialize the reader.
-            reader_kwargs (Dict[str, Any], optional): Keyword arguments to initialize the reader.
-            loader_args (List[Any], optional): Positional arguments for the reader's `load_data` method.
-            loader_kwargs (Dict[str, Any], optional): Keyword arguments for the reader's `load_data` method.
-
-    Returns:
-            Any: The documents or data loaded by the reader.
-
-    Raises:
-            ValueError: If there is an error initializing the reader or loading the data.
-    """
-    try:
-        reader_args = reader_args or []
-        reader_kwargs = reader_kwargs or {}
-        loader_args = loader_args or []
-        loader_kwargs = loader_kwargs or {}
-
-        reader = get_llama_index_reader(reader)
-
-        loader = reader(*reader_args, **reader_kwargs)
-        documents = loader.load_data(*loader_args, **loader_kwargs)
-        return documents
-    except Exception as e:
-        raise ValueError(f"Failed to read and load data. Error: {e}")

lionagi/integrations/bridge/llamaindex_/textnode.py

@@ -1,59 +0,0 @@
-from typing import Any, TypeVar
-
-from lionagi.libs.sys_util import SysUtil
-
-
-def to_llama_index_node(
-    lion_node, node_type: Any = None, **kwargs: Any
-) -> Any:
-    """
-    Converts a Lion node to a Llama Index node of a specified type.
-
-    This function takes a node from the Lion framework, converts it to a dictionary, modifies keys to match
-    the expected Llama Index node schema, and then creates a Llama Index node object of the specified type.
-
-    Args:
-            lion_node: The Lion node to convert. Must have a `to_dict` method.
-            node_type (Any, optional): The type of Llama Index node to create. Can be a string name of a node class
-                    within the Llama Index schema or a class that inherits from `BaseNode`. Defaults to 'TextNode'.
-            **kwargs: Additional keyword arguments to be included in the Llama Index node's initialization.
-
-    Returns:
-            Any: A new instance of the specified Llama Index node type populated with data from the Lion node.
-
-    Raises:
-            TypeError: If `node_type` is neither a string nor a subclass of `BaseNode`.
-            AttributeError: If an error occurs due to an invalid node type or during the creation of the node object.
-    """
-
-    SysUtil.check_import("llama_index", pip_name="llama-index")
-    import llama_index.core.schema
-    from llama_index.core.schema import BaseNode
-
-    node_type = node_type or "TextNode"
-
-    _dict = lion_node.to_dict()
-    SysUtil.change_dict_key(_dict, old_key="content", new_key="text")
-    SysUtil.change_dict_key(_dict, old_key="node_id", new_key="id_")
-    _dict["text"] = str(_dict["text"])
-    _dict = {**_dict, **kwargs}
-
-    if not isinstance(node_type, str) and not issubclass(node_type, BaseNode):
-        raise TypeError(f"node_type must be a string or BaseNode")
-
-    else:
-        try:
-            if isinstance(node_type, str) and hasattr(
-                llama_index.core.schema, node_type
-            ):
-                return getattr(llama_index.core.schema, node_type).from_dict(
-                    _dict
-                )
-            elif issubclass(node_type, BaseNode):
-                return node_type.from_dict(_dict)
-            else:
-                raise AttributeError(
-                    f"Invalid llama-index node type: {node_type}"
-                )
-        except Exception as e:
-            raise AttributeError(f"Error: {e}")

lionagi/integrations/bridge/pydantic_/pydantic_bridge.py

@@ -1,7 +0,0 @@
-from pydantic import (
-    AliasChoices,
-    BaseModel,
-    Field,
-    ValidationError,
-    field_serializer,
-)

lionagi/integrations/bridge/transformers_/install_.py

@@ -1,39 +0,0 @@
-import subprocess
-
-from lionagi.libs import SysUtil
-
-
-def get_pytorch_install_command():
-    cpu_arch = SysUtil.get_cpu_architecture()
-
-    if cpu_arch == "apple_silicon":
-        return "pip install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu"
-    else:
-        # Default CPU installation
-        return "pip install torch torchvision torchaudio"
-
-
-def install_pytorch():
-    command = get_pytorch_install_command()
-    try:
-        subprocess.run(command.split(), check=True)
-        print("PyTorch installed successfully.")
-    except subprocess.CalledProcessError as e:
-        print(f"Failed to install PyTorch: {e}")
-
-
-def install_transformers():
-    if not SysUtil.is_package_installed("torch"):
-        in_ = input(
-            "PyTorch is required for transformers. Would you like to install it now? (y/n): "
-        )
-        if in_ == "y":
-            install_pytorch()
-    if not SysUtil.is_package_installed("transformers"):
-        in_ = input(
-            "transformers is required. Would you like to install it now? (y/n): "
-        )
-        if in_ == "y":
-            SysUtil.install_import(
-                package_name="transformers", import_name="pipeline"
-            )

lionagi/integrations/chunker/chunk.py

@@ -1,314 +0,0 @@
-from collections.abc import Callable
-from typing import Union
-
-from lionagi.core.collections import pile
-from lionagi.core.generic import Node
-from lionagi.libs import func_call
-from lionagi.libs.ln_convert import to_list
-
-from ..bridge.langchain_.langchain_bridge import LangchainBridge
-from ..bridge.llamaindex_.llama_index_bridge import LlamaIndexBridge
-from ..loader.load_util import ChunkerType, _datanode_parser, file_to_chunks
-
-
-def datanodes_convert(documents, chunker_type):
-    """
-    Converts documents to the specified chunker type.
-
-    Args:
-        documents (list): List of documents to be converted.
-        chunker_type (ChunkerType): The type of chunker to convert the documents to.
-
-    Returns:
-        list: The converted documents.
-
-    Example usage:
-        >>> documents = [Node(...), Node(...)]
-        >>> converted_docs = datanodes_convert(documents, ChunkerType.LLAMAINDEX)
-    """
-    for i in range(len(documents)):
-        if type(documents[i]) == Node:
-            if chunker_type == ChunkerType.LLAMAINDEX:
-                documents[i] = documents[i].to_llama_index()
-            elif chunker_type == ChunkerType.LANGCHAIN:
-                documents[i] = documents[i].to_langchain()
-    return documents
-
-
-def text_chunker(documents, args, kwargs):
-    """
-    Chunks text documents into smaller pieces.
-
-    Args:
-        documents (list): List of documents to be chunked.
-        args (tuple): Positional arguments for the chunking function.
-        kwargs (dict): Keyword arguments for the chunking function.
-
-    Returns:
-        pile: A pile of chunked Node instances.
-
-    Example usage:
-        >>> documents = [Node(...), Node(...)]
-        >>> chunked_docs = text_chunker(documents, args, kwargs)
-    """
-
-    def chunk_node(node):
-        chunks = file_to_chunks(node.to_dict(), *args, **kwargs)
-        func_call.lcall(chunks, lambda chunk: chunk.pop("ln_id"))
-        return [Node.from_obj({**chunk}) for chunk in chunks]
-
-    a = to_list(
-        [chunk_node(doc) for doc in documents], flatten=True, dropna=True
-    )
-    return pile(a)
-
-
-def chunk(
-    docs,
-    field: str = "content",
-    chunk_size: int = 1500,
-    overlap: float = 0.1,
-    threshold: int = 200,
-    chunker="text_chunker",
-    chunker_type=ChunkerType.PLAIN,
-    chunker_args=None,
-    chunker_kwargs=None,
-    chunking_kwargs=None,
-    documents_convert_func=None,
-    to_lion: bool | Callable = True,
-):
-    """
-    Chunks documents using the specified chunker.
-
-    Args:
-        docs (list): List of documents to be chunked.
-        field (str, optional): The field to chunk. Defaults to "content".
-        chunk_size (int, optional): The size of each chunk. Defaults to 1500.
-        overlap (float, optional): The overlap between chunks. Defaults to 0.1.
-        threshold (int, optional): The threshold for chunking. Defaults to 200.
-        chunker (str, optional): The chunker function or its name. Defaults to "text_chunker".
-        chunker_type (ChunkerType, optional): The type of chunker to use. Defaults to ChunkerType.PLAIN.
-        chunker_args (list, optional): Positional arguments for the chunker function. Defaults to None.
-        chunker_kwargs (dict, optional): Keyword arguments for the chunker function. Defaults to None.
-        chunking_kwargs (dict, optional): Additional keyword arguments for chunking. Defaults to None.
-        documents_convert_func (Callable, optional): Function to convert documents. Defaults to None.
-        to_lion (bool | Callable, optional): Whether to convert the data to Node instances or a custom parser. Defaults to True.
-
-    Returns:
-        pile: A pile of chunked Node instances.
-
-    Raises:
-        ValueError: If the chunker_type is not supported.
-
-    Example usage:
-        >>> chunked_docs = chunk(docs, field='text', chunk_size=1000, overlap=0.2)
-    """
-
-    if chunker_args is None:
-        chunker_args = []
-    if chunker_kwargs is None:
-        chunker_kwargs = {}
-    if chunking_kwargs is None:
-        chunking_kwargs = {}
-
-    if chunker_type == ChunkerType.PLAIN:
-        chunker_kwargs["field"] = field
-        chunker_kwargs["chunk_size"] = chunk_size
-        chunker_kwargs["overlap"] = overlap
-        chunker_kwargs["threshold"] = threshold
-        return chunk_funcs[ChunkerType.PLAIN](
-            docs, chunker, chunker_args, chunker_kwargs
-        )
-
-    elif chunker_type == ChunkerType.LANGCHAIN:
-        return chunk_funcs[ChunkerType.LANGCHAIN](
-            docs,
-            documents_convert_func,
-            chunker,
-            chunker_args,
-            chunker_kwargs,
-            to_lion,
-        )
-
-    elif chunker_type == ChunkerType.LLAMAINDEX:
-        return chunk_funcs[ChunkerType.LLAMAINDEX](
-            docs,
-            documents_convert_func,
-            chunker,
-            chunker_args,
-            chunker_kwargs,
-            to_lion,
-        )
-
-    elif chunker_type == ChunkerType.SELFDEFINED:
-        return chunk_funcs[ChunkerType.SELFDEFINED](
-            docs,
-            chunker,
-            chunker_args,
-            chunker_kwargs,
-            chunking_kwargs,
-            to_lion,
-        )
-
-    else:
-        raise ValueError(
-            f"{chunker_type} is not supported. Please choose from {list(ChunkerType)}"
-        )
-
-
-def _self_defined_chunker(
-    documents,
-    chunker,
-    chunker_args,
-    chunker_kwargs,
-    chunking_kwargs,
-    to_lion: bool | Callable,
-):
-    """
-    Chunks documents using a self-defined chunker.
-
-    Args:
-        documents (list): List of documents to be chunked.
-        chunker (str | Callable): The chunker function or its name.
-        chunker_args (list): Positional arguments for the chunker function.
-        chunker_kwargs (dict): Keyword arguments for the chunker function.
-        chunking_kwargs (dict): Additional keyword arguments for chunking.
-        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
-
-    Returns:
-        pile: A pile of chunked Node instances or custom parsed nodes.
-
-    Raises:
-        ValueError: If the self-defined chunker is not valid.
-
-    Example usage:
-        >>> chunked_docs = _self_defined_chunker(docs, custom_chunker, ['arg1'], {'key': 'value'}, {}, custom_parser)
-    """
-    try:
-        splitter = chunker(*chunker_args, **chunker_kwargs)
-        nodes = splitter.split(documents, **chunking_kwargs)
-    except Exception as e:
-        raise ValueError(
-            f"Self defined chunker {chunker} is not valid. Error: {e}"
-        ) from e
-
-    if isinstance(to_lion, bool) and to_lion is True:
-        raise ValueError("Please define a valid parser to Node.")
-    elif isinstance(to_lion, Callable):
-        nodes = _datanode_parser(nodes, to_lion)
-    return nodes
-
-
-def _llama_index_chunker(
-    documents,
-    documents_convert_func,
-    chunker,
-    chunker_args,
-    chunker_kwargs,
-    to_lion: bool | Callable,
-):
-    """
-    Chunks documents using a LlamaIndex chunker.
-
-    Args:
-        documents (list): List of documents to be chunked.
-        documents_convert_func (Callable): Function to convert documents.
-        chunker (str | Callable): The chunker function or its name.
-        chunker_args (list): Positional arguments for the chunker function.
-        chunker_kwargs (dict): Keyword arguments for the chunker function.
-        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
-
-    Returns:
-        pile: A pile of chunked Node instances or custom parsed nodes.
-
-    Example usage:
-        >>> chunked_docs = _llama_index_chunker(docs, convert_func, llama_chunker, ['arg1'], {'key': 'value'}, True)
-    """
-    if documents_convert_func:
-        documents = documents_convert_func(documents, "llama_index")
-    nodes = LlamaIndexBridge.llama_index_parse_node(
-        documents, chunker, chunker_args, chunker_kwargs
-    )
-
-    if isinstance(to_lion, bool) and to_lion is True:
-        nodes = [Node.from_llama_index(i) for i in nodes]
-    elif isinstance(to_lion, Callable):
-        nodes = _datanode_parser(nodes, to_lion)
-    return nodes
-
-
-def _langchain_chunker(
-    documents,
-    documents_convert_func,
-    chunker,
-    chunker_args,
-    chunker_kwargs,
-    to_lion: bool | Callable,
-):
-    """
-    Chunks documents using a Langchain chunker.
-
-    Args:
-        documents (list): List of documents to be chunked.
-        documents_convert_func (Callable): Function to convert documents.
-        chunker (str | Callable): The chunker function or its name.
-        chunker_args (list): Positional arguments for the chunker function.
-        chunker_kwargs (dict): Keyword arguments for the chunker function.
-        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
-
-    Returns:
-        pile: A pile of chunked Node instances or custom parsed nodes.
-
-    Example usage:
-        >>> chunked_docs = _langchain_chunker(docs, convert_func, langchain_chunker, ['arg1'], {'key': 'value'}, True)
-    """
-    if documents_convert_func:
-        documents = documents_convert_func(documents, "langchain")
-    nodes = LangchainBridge.langchain_text_splitter(
-        documents, chunker, chunker_args, chunker_kwargs
-    )
-    if isinstance(to_lion, bool) and to_lion is True:
-        if isinstance(documents, str):
-            nodes = [Node(content=i) for i in nodes]
-        else:
-            nodes = [Node.from_langchain(i) for i in nodes]
-    elif isinstance(to_lion, Callable):
-        nodes = _datanode_parser(nodes, to_lion)
-    return nodes
-
-
-def _plain_chunker(documents, chunker, chunker_args, chunker_kwargs):
-    """
-    Chunks documents using a plain chunker.
-
-    Args:
-        documents (list): List of documents to be chunked.
-        chunker (str | Callable): The chunker function or its name.
-        chunker_args (list): Positional arguments for the chunker function.
-        chunker_kwargs (dict): Keyword arguments for the chunker function.
-
-    Returns:
-        pile: A pile of chunked Node instances.
-
-    Raises:
-        ValueError: If the chunker is not supported.
-
-    Example usage:
-        >>> chunked_docs = _plain_chunker(docs, 'text_chunker', ['arg1'], {'key': 'value'})
-    """
-    try:
-        if chunker == "text_chunker":
-            chunker = text_chunker
-        return chunker(documents, chunker_args, chunker_kwargs)
-    except Exception as e:
-        raise ValueError(
-            f"Reader {chunker} is currently not supported. Error: {e}"
-        ) from e
-
-
-chunk_funcs = {
-    ChunkerType.PLAIN: _plain_chunker,
-    ChunkerType.LANGCHAIN: _langchain_chunker,
-    ChunkerType.LLAMAINDEX: _llama_index_chunker,
-    ChunkerType.SELFDEFINED: _self_defined_chunker,
-}

lionagi/integrations/config/__init__.py

@@ -1,4 +0,0 @@
-from .oai_configs import oai_schema
-from .openrouter_configs import openrouter_schema
-
-__all__ = ["oai_schema", "openrouter_schema"]

lionagi/integrations/config/mlx_configs.py

@@ -1 +0,0 @@
-model = "mlx-community/OLMo-7B-hf-4bit-mlx"