lionagi 0.0.209__py3-none-any.whl → 0.0.210__py3-none-any.whl

lionagi/bridge/llama_index.py

@@ -1,266 +0,0 @@
-from typing import Union, Callable, List, Any, Dict, TypeVar
-from ..utils.sys_util import change_dict_key, install_import, is_package_installed
-from ..schema.data_node import DataNode
-
-
-T = TypeVar('T', bound='DataNode')
-
-def from_llama_index(llama_node: Any, **kwargs: Any) -> T:
-    """
-    Converts a Llama Index node into a DataNode object.
-
-    Args:
-        llama_node (Any): The Llama Index node to be converted.
-        **kwargs: Additional keyword arguments for JSON serialization.
-
-    Returns:
-        T: A DataNode object created from the Llama Index node.
-
-    Example:
-        llama_node = LlamaIndexNode(...)
-        datanode = from_llama_index(llama_node, serialize_dates=True)
-    """
-    llama_dict = llama_node.to_dict(**kwargs)
-    return DataNode.from_dict(llama_dict)
-
-def to_llama_index_textnode(datanode: T, **kwargs: Any) -> Any:
-    """
-    Converts a DataNode into a Llama Index TextNode.
-
-    Args:
-        datanode (T): The DataNode to be converted.
-        **kwargs: Additional keyword arguments to be included in the TextNode.
-
-    Returns:
-        Any: A Llama Index TextNode created from the DataNode.
-
-    Example:
-        datanode = DataNode(...)
-        textnode = to_llama_index_textnode(datanode, additional_arg=1)
-    """
-    try:
-        from llama_index.schema import TextNode
-    except ImportError:
-        try:
-            install_import(
-                package_name='llama_index', 
-                module_name='schema',
-                import_name='TextNode'
-            )
-            from llama_index.schema import TextNode
-        except Exception as e:
-            raise ImportError(f'Unable to import required module from llama_index. Please make sure that llama_index is installed. Error: {e}')
-
-    dnode = datanode.to_dict()
-    change_dict_key(dnode, old_key='content', new_key='text')
-    change_dict_key(dnode, old_key='node_id', new_key='id_')
-    dnode['text'] = str(dnode['text'])
-    
-    dnode = {**dnode, **kwargs}
-    return TextNode.from_dict(dnode)
-
-def get_llama_reader(reader: Union[str, Callable]) -> Callable:
-    """
-    Gets a Llama Index reader function.
-
-    Args:
-        reader (Union[str, Callable]): The name of the reader function or the reader function itself.
-
-    Returns:
-        Callable: The Llama Index reader function.
-
-    Raises:
-        ValueError: If the specified reader is invalid.
-
-    Example:
-        reader = get_llama_reader("SimpleDirectoryReader")
-        # or for a custom function
-        def custom_reader(): pass
-        reader = get_llama_reader(custom_reader)
-    """
-    
-    try:
-        if isinstance(reader, str):
-            if reader == 'SimpleDirectoryReader':
-                try:
-                    from llama_index import SimpleDirectoryReader
-                    return SimpleDirectoryReader
-                except ImportError or ModuleNotFoundError:
-                    try:
-                        install_import(
-                            package_name='llama_index',
-                            import_name='SimpleDirectoryReader'
-                        )
-                        from llama_index import SimpleDirectoryReader
-                        return SimpleDirectoryReader
-                    except Exception as e:
-                        raise ImportError(f'Failed to import SimpleDirectoryReader. Error: {e}')
-            else:                
-                try:
-                    from llama_index import download_loader
-                    return download_loader(reader)
-                except ImportError:
-                    try:
-                        install_import(
-                            package_name='llama_index',
-                            import_name='download_loader'
-                        )
-                        return download_loader(reader)
-                    except Exception as e:
-                        raise ImportError(f'Failed to import download_loader from LlamaIndex. Error: {e}')
-        else:
-            return reader
-    except Exception as e:
-        raise ValueError(f'Invalid reader: {reader}, Error: {e}')
-
-def llama_index_reader(reader: Union[str, Callable], 
-                       reader_args: List[Any] = [], 
-                       reader_kwargs: Dict[str, Any] = {}, 
-                       load_data_args: List[Any] = [], 
-                       load_data_kwargs: Dict[str, Any] = {}) -> List[Any]:
-    """
-    Loads documents using a specified Llama Index reader.
-
-    Args:
-        reader (Union[str, Callable]): The name of the reader function or the reader function itself.
-        reader_args (List[Any]): Positional arguments to pass to the reader function.
-        reader_kwargs (Dict[str, Any]): Keyword arguments to pass to the reader function.
-        load_data_args (List[Any]): Positional arguments for the load_data method.
-        load_data_kwargs (Dict[str, Any]): Keyword arguments for the load_data method.
-
-    Returns:
-        List[Any]: A list of documents loaded by the reader.
-
-    Raises:
-        ValueError: If the specified reader is invalid or if the reader fails to load documents.
-
-    Example:
-        documents = llama_index_reader("SimpleDirectoryReader", reader_args=["/path/to/data"])
-    """
-    reader = get_llama_reader(reader)
-
-    try:
-        loader = reader(*reader_args, **reader_kwargs)
-        documents = loader.load_data(*load_data_args, **load_data_kwargs)
-        return documents
-    
-    except Exception as e:
-        raise ValueError(f'Failed to read. Error: {e}')
-
-def get_llama_parser(parser: Union[str, Callable]) -> Callable:
-    """
-    Gets a Llama Index parser function or object.
-
-    Args:
-        parser (Union[str, Callable]): The name of the parser function or the parser function itself.
-
-    Returns:
-        Callable: The Llama Index parser function or object.
-
-    Raises:
-        ValueError: If the specified parser is invalid.
-
-    Example:
-        parser = get_llama_parser("DefaultNodeParser")
-        # or for a custom function
-        def custom_parser(): pass
-        parser = get_llama_parser(custom_parser)
-    """
-    
-    try:
-        import llama_index.node_parser as node_parser
-    except ImportError:
-        try:
-            install_import(
-                package_name='llama_index',
-                module_name='node_parser'
-            )
-            import llama_index.node_parser as node_parser
-        except ImportError:
-            raise ImportError('Failed to import Llama Index. Please install Llama Index to use this function.')
-        except Exception as e:
-            raise ValueError(f'Invalid node parser: {parser}. Error: {e}')
-
-    try:
-        import llama_index.text_splitter as text_splitter
-    except ImportError:
-        try:
-            install_import(
-                package_name='llama_index',
-                module_name='text_splitter'
-            )
-            import llama_index.text_splitter as text_splitter
-        except ImportError:
-            raise ImportError('Failed to import Llama Index. Please install Llama Index to use this function.')
-        
-    try:
-        if parser == 'CodeSplitter':
-            if not is_package_installed('tree_sitter_languages'):
-                install_import(package_name='tree_sitter_languages')
-                
-        a = getattr(node_parser, parser)
-        if a is not None:
-            return a
-        else:
-            raise ImportError(f'Failed to import {parser} from Llama Index.')
-    except Exception as e1:
-        try:
-            if isinstance(parser, str):
-                return getattr(text_splitter, parser)
-            else:
-                return parser
-        except Exception as e2:
-            raise ValueError(f'Invalid node parser: {parser}. Error: {e1}, {e2}')
-
-
-def llama_index_node_parser(documents: List[Any], 
-                            parser: Union[str, Callable], 
-                            parser_args: List[Any] = [], 
-                            parser_kwargs: Dict[str, Any] = {},
-                            parsing_kwargs: Dict[str, Any] = {}) -> List[Any]:
-    """
-    Parses documents into nodes using a specified Llama Index node parser.
-
-    Args:
-        documents (List[Any]): The documents to parse.
-        parser (Union[str, Callable]): The name of the parser function or the parser function itself.
-        parser_args (List[Any]): Positional arguments to pass to the parser function.
-        parser_kwargs (Dict[str, Any]): Keyword arguments to pass to the parser function.
-        parsing_kwargs (Dict[str, Any]): Keyword arguments for the parsing process.
-
-    Returns:
-        List[Any]: A list of nodes parsed from the documents.
-
-    Raises:
-        ValueError: If the specified parser is invalid or if the parser fails to parse the documents.
-
-    Example:
-        nodes = llama_index_node_parser(documents, "DefaultNodeParser")
-    """
-
-    try:
-        parser = get_llama_parser(parser)
-        parser_obj = parser(*parser_args, **parser_kwargs)
-        nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
-        return nodes
-
-    except ImportError as e:
-        module_name = str(e).split("\'")[-2]
-        try:
-            install_import(package_name=module_name)
-            parser = get_llama_parser(parser)
-            parser_obj = parser(*parser_args, **parser_kwargs)
-            nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
-            return nodes
-        except Exception as e:
-            raise ImportError(f'Failed to install and import {module_name}. Error: {e}')
-            
-            
-    except Exception as e1:
-        try:
-            parser_obj = parser.from_defaults(*parser_args, **parser_kwargs)
-            nodes = parser_obj.get_nodes_from_documents(documents, **parsing_kwargs)
-            return nodes
-        except Exception as e2:
-            raise ValueError(f'Failed to parse. Error: {e1}, {e2}')
-        

lionagi/datastores/__init__.py

@@ -1 +0,0 @@
-# TODO

lionagi/datastores/chroma.py

@@ -1 +0,0 @@
-# TODO

lionagi/datastores/deeplake.py

@@ -1 +0,0 @@
-# TODO

lionagi/datastores/elasticsearch.py

@@ -1 +0,0 @@
-# TODO

lionagi/datastores/lantern.py

@@ -1 +0,0 @@
-# TODO

lionagi/datastores/pinecone.py

@@ -1 +0,0 @@
-# TODO

lionagi/datastores/postgres.py

@@ -1 +0,0 @@
-# TODO

lionagi/datastores/qdrant.py

@@ -1 +0,0 @@
-# TODO

lionagi/iservices/anthropic.py

@@ -1,79 +0,0 @@
-from os import getenv
-from .base_service import BaseService, PayloadCreation
-
-class AnthropicService(BaseService):
-    """
-    A service to interact with Anthropic's API endpoints.
-
-    Attributes:
-        base_url (str): The base URL for the Anthropic API.
-        available_endpoints (list): A list of available API endpoints.
-        schema (dict): The schema configuration for the API.
-        key_scheme (str): The environment variable name for Anthropic API key.
-        token_encoding_name (str): The default token encoding scheme.
-
-    Examples:
-        >>> service = AnthropicService(api_key="your_api_key")
-        >>> asyncio.run(service.serve("Hello, world!", "chat/completions"))
-        (payload, completion)
-    """
-
-    base_url = "https://api.anthropic.com/v1/"
-    available_endpoints = ['chat/completions']
-    schema = {}        # TODO
-    key_scheme = "ANTHROPIC_API_KEY"
-    token_encoding_name = "cl100k_base"
-
-    def __init__(self, api_key = None, key_scheme = None,schema = None, token_encoding_name: str = "cl100k_base", **kwargs):
-        key_scheme = key_scheme or self.key_scheme            
-        super().__init__(
-            api_key = api_key or getenv(key_scheme),
-            schema = schema or self.schema,
-            token_encoding_name=token_encoding_name, 
-            **kwargs
-        )
-        self.active_endpoint = []
-
-    async def serve(self, input_, endpoint="chat/completions", method="post", **kwargs):
-        """
-        Serves the input using the specified endpoint and method.
-
-        Args:
-            input_: The input text to be processed.
-            endpoint: The API endpoint to use for processing.
-            method: The HTTP method to use for the request.
-            **kwargs: Additional keyword arguments to pass to the payload creation.
-
-        Returns:
-            A tuple containing the payload and the completion response from the API.
-        """
-        if endpoint not in self.active_endpoint:
-            await self. init_endpoint(endpoint)
-        if endpoint == "chat/completions":
-            return await self.serve_chat(input_, **kwargs)
-        else:
-            return ValueError(f'{endpoint} is currently not supported')
-    
-    async def serve_chat(self, messages, **kwargs):
-        """
-        Serves the chat completion request with the given messages.
-
-        Args:
-            messages: The messages to be included in the chat completion.
-            **kwargs: Additional keyword arguments for payload creation.
-
-        Returns:
-            A tuple containing the payload and the completion response from the API.
-        """
-        if "chat/completions" not in self.active_endpoint:
-            await self. init_endpoint("chat/completions")
-            self.active_endpoint.append("chat/completions")
-        payload = PayloadCreation.chat_completion(
-            messages, self.endpoints["chat/completions"].config, self.schema["chat/completions"], **kwargs)
-
-        try:
-            completion = await self.call_api(payload, "chat/completions", "post")
-            return payload, completion
-        except Exception as e:
-            self.status_tracker.num_tasks_failed += 1
-            raise e

lionagi/iservices/azure.py

@@ -1 +0,0 @@
-# TODO

lionagi/iservices/litellm.py

@@ -1,33 +0,0 @@
-from ..utils.sys_util import install_import, is_package_installed
-from .base_service import BaseService
-
-
-class LiteLLMService(BaseService):
-    def __init__(self, model: str = None, **kwargs):
-        super().__init__()
-
-        try: 
-            if not is_package_installed('litellm'):
-                install_import(
-                    package_name='litellm',
-                    import_name='acompletion'
-                )
-            from litellm import acompletion 
-            self.acompletion = acompletion
-        except:
-            raise ImportError(f'Unable to import required module from ollama. Please make sure that ollama is installed.')
-        
-        self.model = model
-        self.kwargs = kwargs
-
-    async def serve_chat(self, messages, **kwargs):
-        payload = {'messages': messages}
-        kwargs = {**self.kwargs, **kwargs}
-
-        try:
-            completion = await self.acompletion(model=self.model, messages=messages, **kwargs)
-            return payload, completion
-        except Exception as e:
-            self.status_tracker.num_tasks_failed += 1
-            raise e
-        

lionagi/iservices/openrouter.py

@@ -1,44 +0,0 @@
-from os import getenv
-from ..configs.openrouter_configs import openrouter_schema
-from .base_service import BaseService, PayloadCreation
-
-class OpenRouterService(BaseService):
-    base_url = "https://openrouter.ai/api/v1/"
-    available_endpoints = ['chat/completions']
-    schema = openrouter_schema
-    key_scheme = "OPENROUTER_API_KEY"
-    token_encoding_name = "cl100k_base"
-    
-    
-    def __init__(self, api_key = None, key_scheme = None,schema = None, token_encoding_name: str = "cl100k_base", **kwargs):
-        key_scheme = key_scheme or self.key_scheme            
-        super().__init__(
-            api_key = api_key or getenv(key_scheme),
-            schema = schema or self.schema,
-            token_encoding_name=token_encoding_name, **kwargs
-        )
-        self.active_endpoint = []
-
-    async def serve(self, input_, endpoint="chat/completions", method="post", **kwargs):
-        if endpoint not in self.active_endpoint:
-            await self. init_endpoint(endpoint)
-        if endpoint == "chat/completions":
-            return await self.serve_chat(input_, **kwargs)
-        else:
-            return ValueError(f'{endpoint} is currently not supported')
-    
-    async def serve_chat(self, messages, **kwargs):
-        endpoint = "chat/completions"
-        
-        if endpoint not in self.active_endpoint:
-            await self. init_endpoint(endpoint)
-            self.active_endpoint.append(endpoint)
-        payload = PayloadCreation.chat_completion(
-            messages, self.endpoints[endpoint].config, self.schema[endpoint], **kwargs)
-
-        try:
-            completion = await self.call_api(payload, endpoint, "post")
-            return payload, completion
-        except Exception as e:
-            self.status_tracker.num_tasks_failed += 1
-            raise e

lionagi/loaders/__init__.py

@@ -1,18 +0,0 @@
-from .load_util import dir_to_path, dir_to_nodes,  chunk_text, read_text, file_to_chunks
-from .reader import load, ReaderType, text_reader
-from .chunker import chunk, datanodes_convert, ChunkerType, text_chunker
-
-__all__ = [
-    'load',
-    'chunk',
-    'datanodes_convert',
-    'text_reader',
-    'text_chunker',
-    'ReaderType',
-    'ChunkerType',
-    'dir_to_path', 
-    'dir_to_nodes', 
-    'chunk_text', 
-    'read_text', 
-    'file_to_chunks'
-]

lionagi/loaders/chunker.py

@@ -1,166 +0,0 @@
-from typing import Union, Callable
-
-from ..utils import lcall
-from ..schema import DataNode
-from ..bridge import langchain_text_splitter, from_langchain, llama_index_node_parser, from_llama_index
-from .load_util import ChunkerType, file_to_chunks
-
-
-def datanodes_convert(documents, chunker_type):
-    """
-    Converts a list of DataNode documents to a specific format based on the chunker type.
-
-    Args:
-        documents: A list of DataNode instances to be converted.
-        chunker_type: The chunker type to determine the conversion format.
-
-    Returns:
-        The list of converted DataNode instances.
-
-    Example usage:
-        >>> documents = [DataNode(content="Example content")]
-        >>> converted = datanodes_convert(documents, ChunkerType.LANGCHAIN)
-    """
-    for i in range(len(documents)):
-        if type(documents[i]) == DataNode:
-            if chunker_type == ChunkerType.LLAMAINDEX:
-                documents[i] = documents[i].to_llama_index()
-            elif chunker_type == ChunkerType.LANGCHAIN:
-                documents[i] = documents[i].to_langchain()
-    return documents
-
-def text_chunker(documents, args, kwargs):
-    """
-    Chunks text documents into smaller pieces.
-
-    Args:
-        documents: A list of DataNode instances to be chunked.
-        args: Positional arguments to be passed to the chunking function.
-        kwargs: Keyword arguments to be passed to the chunking function.
-
-    Returns:
-        A list of chunked DataNode instances.
-
-    Example usage:
-        >>> documents = [DataNode(content="Example content")]
-        >>> args = []
-        >>> kwargs = {"chunk_size": 100}
-        >>> chunked_docs = text_chunker(documents, args, kwargs)
-    """
-    def chunk_node(node):
-        chunks = file_to_chunks(node.to_dict(), *args, **kwargs)
-        lcall(chunks, lambda chunk: chunk.pop('node_id'))
-        chunk_nodes = lcall(chunks, lambda x: DataNode(**x))
-        return chunk_nodes
-
-    nodes = []
-    for doc in documents:
-        nodes += chunk_node(doc)
-    return nodes
-
-
-def _datanode_parser(nodes, parser):
-    """
-    Parses raw data into DataNode instances using the provided parser function.
-
-    Args:
-        nodes: A list of raw data to be parsed.
-        parser: A function that parses raw data into DataNode instances.
-
-    Returns:
-        A list of parsed DataNode instances.
-
-    Raises:
-        ValueError: If the parser function fails.
-
-    Example usage:
-        >>> raw_data = [{"content": "Example content"}]
-        >>> parser = lambda x: [DataNode(**d) for d in x]
-        >>> parsed_nodes = _datanode_parser(raw_data, parser)
-    """
-    try:
-        nodes = parser(nodes)
-    except Exception as e:
-        raise ValueError(f'DataNode parser {parser} failed. Error:{e}')
-    return nodes
-
-
-def chunk(documents,
-          chunker,
-          chunker_type=ChunkerType.PLAIN,
-          chunker_args=[],
-          chunker_kwargs={},
-          chunking_kwargs={},
-          documents_convert_func=None,
-          to_datanode: Union[bool, Callable] = True):
-    """
-    Chunks documents using the specified chunker and chunker type.
-
-    Args:
-        documents: A list of documents to be chunked.
-        chunker: The chunking function to be used.
-        chunker_type: The type of the chunker. Defaults to ChunkerType.PLAIN.
-        chunker_args: Positional arguments for the chunker function. Defaults to an empty list.
-        chunker_kwargs: Keyword arguments for the chunker function. Defaults to an empty dict.
-        chunking_kwargs: Additional keyword arguments for the chunking process. Defaults to an empty dict.
-        documents_convert_func: A function to convert documents to a specific format. Defaults to None.
-        to_datanode: Determines whether to convert the result into DataNode instances, or a callable to convert the result. Defaults to True.
-
-    Returns:
-        A list of chunked DataNode instances after applying the chunker.
-
-    Raises:
-        ValueError: If the chunker fails or an unsupported chunker type is provided.
-
-    Example usage:
-        >>> documents = ["Long text document...", "Another long text..."]
-        >>> chunked_docs = chunk(documents, text_chunker, ChunkerType.PLAIN, chunker_args=[], chunker_kwargs={"chunk_size": 100})
-    """
-    if chunker_type == ChunkerType.PLAIN:
-        try:
-            if chunker == 'text_chunker':
-                chunker = text_chunker
-            nodes = chunker(documents, chunker_args, chunker_kwargs)
-            return nodes
-        except Exception as e:
-            raise ValueError(f'Reader {chunker} is currently not supported. Error: {e}')
-    if chunker_type == ChunkerType.LANGCHAIN:
-        if documents_convert_func:
-            documents = documents_convert_func(documents, 'langchain')
-        nodes = langchain_text_splitter(documents, chunker, chunker_args, chunker_kwargs)
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            if isinstance(documents, str):
-                nodes = lcall(nodes, lambda x: DataNode(content=x))
-            else:
-                nodes = lcall(nodes, from_langchain)
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    elif chunker_type == ChunkerType.LLAMAINDEX:
-        if documents_convert_func:
-            documents = documents_convert_func(documents, 'llama_index')
-        nodes = llama_index_node_parser(documents, chunker, chunker_args, chunker_kwargs, chunking_kwargs)    
-            
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            nodes = lcall(nodes, from_llama_index)
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    elif chunker_type == ChunkerType.SELFDEFINED:
-        try:
-            splitter = chunker(*chunker_args, **chunker_kwargs)
-            nodes = splitter.split(documents, **chunking_kwargs)
-        except Exception as e:
-            raise ValueError(f'Self defined chunker {chunker} is not valid. Error: {e}')
-
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            raise ValueError(f'Please define a valid parser to DataNode.')
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    else:
-        raise ValueError(f'{chunker_type} is not supported. Please choose from {list(ChunkerType)}')
-