lionagi 0.0.208__py3-none-any.whl → 0.0.210__py3-none-any.whl

lionagi/loaders/chunker.py

@@ -1,166 +0,0 @@
-from typing import Union, Callable
-
-from ..utils import lcall
-from ..schema import DataNode
-from ..bridge import langchain_text_splitter, from_langchain, llama_index_node_parser, from_llama_index
-from .load_util import ChunkerType, file_to_chunks
-
-
-def datanodes_convert(documents, chunker_type):
-    """
-    Converts a list of DataNode documents to a specific format based on the chunker type.
-
-    Args:
-        documents: A list of DataNode instances to be converted.
-        chunker_type: The chunker type to determine the conversion format.
-
-    Returns:
-        The list of converted DataNode instances.
-
-    Example usage:
-        >>> documents = [DataNode(content="Example content")]
-        >>> converted = datanodes_convert(documents, ChunkerType.LANGCHAIN)
-    """
-    for i in range(len(documents)):
-        if type(documents[i]) == DataNode:
-            if chunker_type == ChunkerType.LLAMAINDEX:
-                documents[i] = documents[i].to_llama_index()
-            elif chunker_type == ChunkerType.LANGCHAIN:
-                documents[i] = documents[i].to_langchain()
-    return documents
-
-def text_chunker(documents, args, kwargs):
-    """
-    Chunks text documents into smaller pieces.
-
-    Args:
-        documents: A list of DataNode instances to be chunked.
-        args: Positional arguments to be passed to the chunking function.
-        kwargs: Keyword arguments to be passed to the chunking function.
-
-    Returns:
-        A list of chunked DataNode instances.
-
-    Example usage:
-        >>> documents = [DataNode(content="Example content")]
-        >>> args = []
-        >>> kwargs = {"chunk_size": 100}
-        >>> chunked_docs = text_chunker(documents, args, kwargs)
-    """
-    def chunk_node(node):
-        chunks = file_to_chunks(node.to_dict(), *args, **kwargs)
-        lcall(chunks, lambda chunk: chunk.pop('node_id'))
-        chunk_nodes = lcall(chunks, lambda x: DataNode(**x))
-        return chunk_nodes
-
-    nodes = []
-    for doc in documents:
-        nodes += chunk_node(doc)
-    return nodes
-
-
-def _datanode_parser(nodes, parser):
-    """
-    Parses raw data into DataNode instances using the provided parser function.
-
-    Args:
-        nodes: A list of raw data to be parsed.
-        parser: A function that parses raw data into DataNode instances.
-
-    Returns:
-        A list of parsed DataNode instances.
-
-    Raises:
-        ValueError: If the parser function fails.
-
-    Example usage:
-        >>> raw_data = [{"content": "Example content"}]
-        >>> parser = lambda x: [DataNode(**d) for d in x]
-        >>> parsed_nodes = _datanode_parser(raw_data, parser)
-    """
-    try:
-        nodes = parser(nodes)
-    except Exception as e:
-        raise ValueError(f'DataNode parser {parser} failed. Error:{e}')
-    return nodes
-
-
-def chunk(documents,
-          chunker,
-          chunker_type=ChunkerType.PLAIN,
-          chunker_args=[],
-          chunker_kwargs={},
-          chunking_kwargs={},
-          documents_convert_func=None,
-          to_datanode: Union[bool, Callable] = True):
-    """
-    Chunks documents using the specified chunker and chunker type.
-
-    Args:
-        documents: A list of documents to be chunked.
-        chunker: The chunking function to be used.
-        chunker_type: The type of the chunker. Defaults to ChunkerType.PLAIN.
-        chunker_args: Positional arguments for the chunker function. Defaults to an empty list.
-        chunker_kwargs: Keyword arguments for the chunker function. Defaults to an empty dict.
-        chunking_kwargs: Additional keyword arguments for the chunking process. Defaults to an empty dict.
-        documents_convert_func: A function to convert documents to a specific format. Defaults to None.
-        to_datanode: Determines whether to convert the result into DataNode instances, or a callable to convert the result. Defaults to True.
-
-    Returns:
-        A list of chunked DataNode instances after applying the chunker.
-
-    Raises:
-        ValueError: If the chunker fails or an unsupported chunker type is provided.
-
-    Example usage:
-        >>> documents = ["Long text document...", "Another long text..."]
-        >>> chunked_docs = chunk(documents, text_chunker, ChunkerType.PLAIN, chunker_args=[], chunker_kwargs={"chunk_size": 100})
-    """
-    if chunker_type == ChunkerType.PLAIN:
-        try:
-            if chunker == 'text_chunker':
-                chunker = text_chunker
-            nodes = chunker(documents, chunker_args, chunker_kwargs)
-            return nodes
-        except Exception as e:
-            raise ValueError(f'Reader {chunker} is currently not supported. Error: {e}')
-    if chunker_type == ChunkerType.LANGCHAIN:
-        if documents_convert_func:
-            documents = documents_convert_func(documents, 'langchain')
-        nodes = langchain_text_splitter(documents, chunker, chunker_args, chunker_kwargs)
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            if isinstance(documents, str):
-                nodes = lcall(nodes, lambda x: DataNode(content=x))
-            else:
-                nodes = lcall(nodes, from_langchain)
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    elif chunker_type == ChunkerType.LLAMAINDEX:
-        if documents_convert_func:
-            documents = documents_convert_func(documents, 'llama_index')
-        nodes = llama_index_node_parser(documents, chunker, chunker_args, chunker_kwargs, chunking_kwargs)    
-            
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            nodes = lcall(nodes, from_llama_index)
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    elif chunker_type == ChunkerType.SELFDEFINED:
-        try:
-            splitter = chunker(*chunker_args, **chunker_kwargs)
-            nodes = splitter.split(documents, **chunking_kwargs)
-        except Exception as e:
-            raise ValueError(f'Self defined chunker {chunker} is not valid. Error: {e}')
-
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            raise ValueError(f'Please define a valid parser to DataNode.')
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    else:
-        raise ValueError(f'{chunker_type} is not supported. Please choose from {list(ChunkerType)}')
-    

lionagi/loaders/load_util.py

@@ -1,240 +0,0 @@
-# use utils and schema
-import math
-from enum import Enum
-from pathlib import Path
-from typing import List, Union, Dict, Any, Tuple
-
-from ..utils import to_list, lcall
-from ..schema import DataNode
-
-class ReaderType(str, Enum):
-    PLAIN = 'plain'
-    LANGCHAIN = 'langchain'
-    LLAMAINDEX = 'llama_index'
-    SELFDEFINED = 'self_defined'
-
-
-class ChunkerType(str, Enum):
-    PLAIN = 'plain'                 # default
-    LANGCHAIN = 'langchain'         # using langchain functions
-    LLAMAINDEX = 'llama_index'      # using llamaindex functions
-    SELFDEFINED = 'self_defined'    # create custom functions
-    
-    
-def dir_to_path(
-    dir: str, ext: str, recursive: bool = False, 
-    flatten: bool = True
-) -> List[Path]:
-    """
-    Generates a list of file paths from a directory with the given file extension.
-
-    Parameters:
-        dir (str): The directory to search for files.
-
-        ext (str): The file extension to filter by.
-
-        recursive (bool): Whether to search subdirectories recursively. Defaults to False.
-
-        flatten (bool): Whether to flatten the list. Defaults to True.
-    
-    Returns:
-        List[Path]: A list of Paths to the files.
-
-    Raises:
-        ValueError: If the directory or extension is invalid.
-    """
-    
-    def _dir_to_path(ext):
-        tem = '**/*' if recursive else '*'
-        return list(Path(dir).glob(tem + ext))
-    
-    try: 
-        return to_list(lcall(ext, _dir_to_path, flatten=True), flatten=flatten)
-    except: 
-        raise ValueError("Invalid directory or extension, please check the path")
-
-def dir_to_nodes(
-    dir: str, ext: Union[List[str], str], 
-    recursive: bool = False, flatten: bool = True, 
-    clean_text: bool = True
-) -> List[DataNode]:
-    """
-    Converts directory contents into DataNode objects based on specified file extensions.
-
-    This function first retrieves a list of file paths from the specified directory, matching the given file extension. It then reads the content of these files, optionally cleaning the text, and converts each file's content into a DataNode object. 
-
-    Parameters:
-        dir (str): The directory path from which to read files.
-        ext: The file extension(s) to include. Can be a single string or a list/tuple of strings.
-        recursive (bool, optional): If True, the function searches for files recursively in subdirectories. Defaults to False.
-        flatten (bool, optional): If True, flattens the directory structure in the returned paths. Defaults to True.
-        clean_text (bool, optional): If True, cleans the text read from files. Defaults to True.
-
-    Returns:
-        list: A list of DataNode objects created from the files in the specified directory.
-
-    Example:
-        nodes = dir_to_nodes("/path/to/dir", ".txt", recursive=True)
-        # This would read all .txt files in /path/to/dir and its subdirectories, 
-        # converting them into DataNode objects.
-    """
-
-    path_list = dir_to_path(dir, ext, recursive, flatten)
-    files_info = lcall(path_list, read_text, clean=clean_text)
-    nodes = lcall(files_info, lambda x: DataNode(content=x[0], metadata=x[1]))
-    return nodes
-
-def chunk_text(input: str, 
-               chunk_size: int, 
-               overlap: float,
-               threshold: int) -> List[Union[str, None]]:
-    """
-    Chunks the input text into smaller parts, with optional overlap and threshold for final chunk.
-
-    Parameters:
-        input (str): The input text to chunk.
-
-        chunk_size (int): The size of each chunk.
-
-        overlap (float): The amount of overlap between chunks.
-
-        threshold (int): The minimum size of the final chunk.
-
-    Returns:
-        List[Union[str, None]]: A list of text chunks.
-
-    Raises:
-        ValueError: If an error occurs during chunking.
-    """
-    
-    def _chunk_n1():
-        return [input]
-        
-    def _chunk_n2():
-        chunks = []
-        chunks.append(input[:chunk_size + overlap_size])
-        
-        if len(input) - chunk_size > threshold:
-            chunks.append(input[chunk_size - overlap_size:])    
-        else:
-            return _chunk_n1()
-        
-        return chunks
-
-    def _chunk_n3():
-        chunks = []
-        chunks.append(input[:chunk_size + overlap_size])
-        for i in range(1, n_chunks - 1):
-            start_idx = chunk_size * i - overlap_size
-            end_idx = chunk_size * (i + 1) + overlap_size
-            chunks.append(input[start_idx:end_idx])
-
-        if len(input) - chunk_size * (n_chunks - 1) > threshold:
-            chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size:])
-        else:
-            chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size:]
-
-        return chunks
-    
-    try:
-        if not isinstance(input, str): input = str(input)
-        
-        n_chunks = math.ceil(len(input) / chunk_size)
-        overlap_size = int(overlap / 2)
-
-        if n_chunks == 1: 
-            return _chunk_n1()
-        
-        elif n_chunks == 2:
-            return _chunk_n2()
-        
-        elif n_chunks > 2:
-            return _chunk_n3()
-
-    except Exception as e:
-        raise ValueError(f"An error occurred while chunking the text. {e}")
-
-def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
-    """
-    Reads text from a file and optionally cleans it, returning the content and metadata.
-
-    Parameters:
-        filepath (str): The path to the file to read.
-
-        clean (bool): Whether to clean the text by replacing certain characters. Defaults to True.
-
-    Returns:
-        Tuple[str, dict]: A tuple containing the content and metadata of the file.
-
-    Raises:
-        FileNotFoundError: If the file cannot be found.
-
-        PermissionError: If there are permissions issues.
-
-        OSError: For other OS-related errors.
-    """
-    def _get_metadata():
-        import os
-        from datetime import datetime
-        file = filepath
-        size = os.path.getsize(filepath)
-        creation_date = datetime.fromtimestamp(os.path.getctime(filepath)).date()
-        modified_date = datetime.fromtimestamp(os.path.getmtime(filepath)).date()
-        last_accessed_date = datetime.fromtimestamp(os.path.getatime(filepath)).date()
-        return {'file': str(file),
-                'size': size,
-                'creation_date': str(creation_date),
-                'modified_date': str(modified_date),
-                'last_accessed_date': str(last_accessed_date)}
-    try:
-        with open(filepath, 'r') as f:
-            content = f.read()
-            if clean:
-                # Define characters to replace and their replacements
-                replacements = {'\\': ' ', '\n': ' ', '\t': ' ', '  ': ' ', '\'': ' '}
-                for old, new in replacements.items():
-                    content = content.replace(old, new)
-            metadata = _get_metadata()
-            return content, metadata
-    except Exception as e:
-        raise e
-    
-def _file_to_chunks(input: Dict[str, Any],
-                   field: str = 'content',
-                   chunk_size: int = 1500,
-                   overlap: float = 0.1,
-                   threshold: int = 200) -> List[Dict[str, Any]]:
-    try:
-        out = {key: value for key, value in input.items() if key != field}
-        out.update({"chunk_overlap": overlap, "chunk_threshold": threshold})
-
-        chunks = chunk_text(input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold)
-        logs = []
-        for i, chunk in enumerate(chunks):
-            chunk_dict = out.copy()
-            chunk_dict.update({
-                'file_chunks': len(chunks),
-                'chunk_id': i + 1,
-                'chunk_size': len(chunk),
-                f'chunk_{field}': chunk
-            })
-            logs.append(chunk_dict)
-
-        return logs
-
-    except Exception as e:
-        raise ValueError(f"An error occurred while chunking the file. {e}")
-
-
-# needs doing TODO
-def file_to_chunks(input,
-                #    project='project',
-                #    output_dir='data/logs/sources/',
-                   chunk_func = _file_to_chunks,  **kwargs):
-                #    out_to_csv=False,
-                #    filename=None,
-                #    verbose=True,
-                #    timestamp=True,
-                #    logger=None,
-    logs = to_list(lcall(input, chunk_func, **kwargs), flatten=True)
-    return logs

lionagi/loaders/reader.py

@@ -1,122 +0,0 @@
-from typing import Union, Callable
-
-from ..utils import lcall
-from ..bridge import langchain_loader, from_langchain, llama_index_reader, from_llama_index
-from .load_util import dir_to_nodes, ReaderType
-
-
-def _datanode_parser(nodes, parser):
-    """
-    Parses raw data into DataNode instances using the provided parser function.
-
-    Args:
-        nodes: The list of raw data to be parsed.
-        parser: The parser function to transform nodes into DataNode instances.
-
-    Returns:
-        A list of parsed DataNode instances.
-
-    Raises:
-        ValueError: If the parser function fails.
-
-    Example usage:
-        >>> raw_nodes = [{'content': 'Example content'}]
-        >>> parser = lambda x: [DataNode(**node) for node in x]
-        >>> datanodes = _datanode_parser(raw_nodes, parser)
-    """
-    try:
-        nodes = parser(nodes)
-    except Exception as e:
-        raise ValueError(f'DataNode parser {parser} failed. Error:{e}')
-    return nodes
-
-def text_reader(args, kwargs):
-    """
-    Reads text files from a directory and converts them to DataNode instances.
-
-    Args:
-        args: Positional arguments for the dir_to_nodes function.
-        kwargs: Keyword arguments for the dir_to_nodes function.
-
-    Returns:
-        A list of DataNode instances.
-
-    Example usage:
-        >>> args = ['path/to/text/files']
-        >>> kwargs = {'file_extension': 'txt'}
-        >>> nodes = text_reader(args, kwargs)
-    """
-    return dir_to_nodes(*args, **kwargs)
-
-
-def load(reader: Union[str, Callable],
-         reader_type=ReaderType.PLAIN,
-         reader_args=[],
-         reader_kwargs={},
-         load_args=[],
-         load_kwargs={},
-         to_datanode: Union[bool, Callable] = True):
-    """
-    Loads documents using the specified reader and reader type.
-
-    Args:
-        reader: The reader function or its name as a string.
-        reader_type: The type of the reader. Defaults to ReaderType.PLAIN.
-        reader_args: Positional arguments for the reader function. Defaults to an empty list.
-        reader_kwargs: Keyword arguments for the reader function. Defaults to an empty dict.
-        load_args: Positional arguments for the loader function. Defaults to an empty list.
-        load_kwargs: Keyword arguments for the loader function. Defaults to an empty dict.
-        to_datanode: Determines whether to convert the result into DataNode instances, or
-                     a callable to convert the result. Defaults to True.
-
-    Returns:
-        A list of loaded and potentially parsed documents.
-
-    Raises:
-        ValueError: If the reader fails or an unsupported reader type is provided.
-
-    Example usage:
-        >>> reader = 'text_reader'
-        >>> reader_args = ['path/to/text/files']
-        >>> reader_kwargs = {'file_extension': 'txt'}
-        >>> nodes = load(reader, reader_args=reader_args, reader_kwargs=reader_kwargs)
-    """
-    if reader_type == ReaderType.PLAIN:
-        try:
-            if reader == 'text_reader':
-                reader = text_reader
-            nodes = reader(reader_args, reader_kwargs)
-            return nodes
-        except Exception as e:
-            raise ValueError(f'Reader {reader} is currently not supported. Error: {e}')
-    if reader_type == ReaderType.LANGCHAIN:
-        nodes = langchain_loader(reader, reader_args, reader_kwargs)
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            nodes = lcall(nodes, from_langchain)
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    elif reader_type == ReaderType.LLAMAINDEX:
-        nodes = llama_index_reader(reader, reader_args, reader_kwargs, load_args, load_kwargs)
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            nodes = lcall(nodes, from_llama_index)
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    elif reader_type == ReaderType.SELFDEFINED:
-        try:
-            loader = reader(*reader_args, **reader_kwargs)
-            nodes = loader.load(*load_args, **load_kwargs)
-        except Exception as e:
-            raise ValueError(f'Self defined reader {reader} is not valid. Error: {e}')
-
-        if isinstance(to_datanode, bool) and to_datanode is True:
-            raise ValueError(f'Please define a valid parser to DataNode.')
-        elif isinstance(to_datanode, Callable):
-            nodes = _datanode_parser(nodes, to_datanode)
-        return nodes
-
-    else:
-        raise ValueError(f'{reader_type} is not supported. Please choose from {list(ReaderType)}')

lionagi/models/imodel.py

@@ -1,53 +0,0 @@
-# from ..configs import oai_schema
-
-# class BaseIntelligentModel:
-    
-#     def __init__(
-#         self, 
-#         service=None, 
-#         default_config=oai_schema['chat']['config'], 
-#         **kwargs
-#     ) -> None:
-#     # kwargs are the individual parameters that the model can take
-#     # different for different models
-#         self.service=service, 
-#         self.config = {**default_config, **kwargs},
-
-#     async def __call__(
-#         self, 
-#         payload, 
-#         service=None, 
-#         endpoint_='chat/completions', 
-#         method='post'
-#     ):
-#         service = service or self.service
-#         return await service.serve(
-#             payload=payload, endpoint_=endpoint_, method=method
-#         )
-
-#     def set_service(self, service):
-#         self.service = service
-
-#     def set_config(self, config):
-#         self.config=config
-
-#     def change_model(self, model):
-#         self.config['model'] = model
-        
-#     def change_temperature(self, temperature):
-#         self.config['temperature'] = temperature
-    
-#     def revert_to_default_config(self):
-#         self.config = oai_schema['chat']['config']
-    
-#     def modify_config(self, **kwargs):
-#         self.config = {**self.config, **kwargs}
-    
-
-# class IModel(BaseIntelligentModel):
-    
-#     def __init__(
-#         self, service=None, default_model_kwargs=None, **kwargs
-#     ) -> None:
-#         super().__init__(service, default_model_kwargs, **kwargs)
-