lionagi 0.0.111__py3-none-any.whl → 0.0.113__py3-none-any.whl

lionagi/utils/doc_util.py

@@ -1,331 +0,0 @@
-import math
-from pathlib import Path
-from typing import Any, Dict, List, Union, Callable, Optional
-
-from .sys_util import to_list, l_call
-from .log_util import DataLogger
-
-
-def dir_to_path(dir: str, ext, recursive: bool = False, flat: bool = True):
-    """
-    Retrieves a list of file paths in the specified directory with the given extension.
-
-    Parameters:
-        dir (str): The directory path where to search for files.
-
-        ext (str): The file extension to filter by.
-
-        recursive (bool, optional): If True, search for files recursively in subdirectories. Defaults to False.
-
-        flat (bool, optional): If True, return a flat list of file paths. Defaults to True.
-
-    Returns:
-        List[str]: A list of file paths that match the given extension within the specified directory.
-
-    Example:
-        >>> files = dir_to_path(dir='my_directory', ext='.txt', recursive=True, flat=True)
-    """
-
-    def _dir_to_path(ext, recursive=recursive):
-        tem = '**/*' if recursive else '*'
-        return list(Path(dir).glob(tem + ext))
-
-    try: 
-        return to_list(l_call(ext, _dir_to_path, flat=True), flat=flat)
-    except: 
-        raise ValueError("Invalid directory or extension, please check the path")
-    
-def read_text(filepath: str, clean: bool = True) -> str:
-    """
-    Reads the content of a text file and optionally cleans it by removing specified characters.
-
-    Parameters:
-        filepath (str): The path to the text file to be read.
-
-        clean (bool, optional): If True, clean the content by removing specific unwanted characters. Defaults to True.
-
-    Returns:
-        str: The cleaned (if 'clean' is True) or raw content of the text file.
-
-    Example:
-        >>> content = read_text(filepath='example.txt', clean=True)
-    """
-
-    with open(filepath, 'r') as f:
-        content = f.read()
-        if clean:
-            # Define characters to replace and their replacements
-            replacements = {'\\': ' ', '\n': ' ', '\t': ' ', '  ': ' ', '\'': ' '}
-            for old, new in replacements.items():
-                content = content.replace(old, new)
-        return content
-
-def dir_to_files(dir: str, ext: str, recursive: bool = False,
-                 reader: Callable = read_text, clean: bool = True,
-                 to_csv: bool = False, project: str = 'project',
-                 output_dir: str = 'data/logs/sources/', filename: Optional[str] = None,
-                 verbose: bool = True, timestamp: bool = True, logger: Optional[DataLogger] = None):
-    """
-    Reads and processes files in a specified directory with the given extension.
-
-    Parameters:
-        dir (str): The directory path where files are located.
-
-        ext (str): The file extension to filter by.
-
-        recursive (bool, optional): If True, search files recursively in subdirectories. Defaults to False.
-
-        reader (Callable, optional): Function used to read and process the content of each file. Defaults to read_text.
-
-        clean (bool, optional): If True, cleans the content by removing specified characters. Defaults to True.
-
-        to_csv (bool, optional): If True, export the processed data to a CSV file. Defaults to False.
-
-        project (str, optional): The name of the project. Defaults to 'project'.
-
-        output_dir (str, optional): Directory path for exporting the CSV file. Defaults to 'data/logs/sources/'.
-
-        filename (Optional[str], optional): Name of the CSV file, if not provided, a default will be used. Defaults to None.
-
-        verbose (bool, optional): If True, print a message upon CSV export. Defaults to True.
-
-        timestamp (bool, optional): If True, include a timestamp in the file name. Defaults to True.
-
-        logger (Optional[DataLogger], optional): An instance of DataLogger for logging, if not provided, a new one will be created. Defaults to None.
-
-    Returns:
-        List[Dict[str, Union[str, Path]]]: A list of dictionaries containing file information and content.
-
-    Examples:
-        >>> logs = dir_to_files(dir='my_directory', ext='.txt', to_csv=True)
-    """
-
-    sources = dir_to_path(dir, ext, recursive)
-
-    def _split_path(path: Path) -> tuple:
-        folder_name = path.parent.name
-        file_name = path.name
-        return (folder_name, file_name)
-
-    def _to_dict(path_: Path) -> Dict[str, Union[str, Path]]:
-        folder, file = _split_path(path_)
-        content = reader(str(path_), clean=clean)
-        return {
-            'project': project,
-            'folder': folder,
-            'file': file,
-            "file_size": len(str(content)),
-            'content': content
-        } if content else None
-
-    logs = to_list(l_call(sources, _to_dict, flat=True), dropna=True)
-
-    if to_csv:
-        filename = filename or f"{project}_sources.csv"
-        logger = DataLogger(dir=output_dir, log=logs) if not logger else logger
-        logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
-
-    return logs
-
-def chunk_text(input: str, chunk_size: int, overlap: float,
-               threshold: int) -> List[Union[str, None]]:
-    """
-    Splits a string into chunks of a specified size, allowing for optional overlap between chunks.
-    
-    Parameters:
-        input (str): The text to be split into chunks.
-
-        chunk_size (int): The size of each chunk in characters.
-
-        overlap (float): A value between [0, 1] specifying the percentage of overlap between adjacent chunks.
-
-        threshold (int): The minimum size for the last chunk. If the last chunk is smaller than this, it will be merged with the previous chunk.
-        
-    Raises:
-        TypeError: If input text cannot be converted to a string.
-
-        ValueError: If any error occurs during the chunking process.
-        
-    Returns:
-        List[Union[str, None]]: List of text chunks.
-    """
-
-    try:
-        # Ensure text is a string
-        if not isinstance(input, str):
-            input = str(input)
-
-        chunks = []
-        n_chunks = math.ceil(len(input) / chunk_size)
-        overlap_size = int(chunk_size * overlap / 2)
-
-        if n_chunks == 1:
-            return [input]
-
-        elif n_chunks == 2:
-            chunks.append(input[:chunk_size + overlap_size])
-            if len(input) - chunk_size > threshold:
-                chunks.append(input[chunk_size - overlap_size:])
-            else:
-                return [input]
-            return chunks
-
-        elif n_chunks > 2:
-            chunks.append(input[:chunk_size + overlap_size])
-            for i in range(1, n_chunks - 1):
-                start_idx = chunk_size * i - overlap_size
-                end_idx = chunk_size * (i + 1) + overlap_size
-                chunks.append(input[start_idx:end_idx])
-
-            if len(input) - chunk_size * (n_chunks - 1) > threshold:
-                chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size:])
-            else:
-                chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size:]
-
-            return chunks
-
-    except Exception as e:
-        raise ValueError(f"An error occurred while chunking the text. {e}")
-
-def _file_to_chunks(input: Dict[str, Any],
-                   field: str = 'content',
-                   chunk_size: int = 1500,
-                   overlap: float = 0.2,
-                   threshold: int = 200) -> List[Dict[str, Any]]:
-    """
-    Splits text from a specified dictionary field into chunks and returns a list of dictionaries.
-    
-    Parameters:
-        input (Dict[str, Any]): The input dictionary containing the text field to be chunked.
-
-        field (str, optional): The dictionary key corresponding to the text field. Defaults to 'content'.
-
-        chunk_size (int, optional): Size of each text chunk in characters. Defaults to 1500.
-
-        overlap (float, optional): Percentage of overlap between adjacent chunks, in the range [0, 1]. Defaults to 0.2.
-
-        threshold (int, optional): Minimum size for the last chunk. If smaller, it will be merged with the previous chunk. Defaults to 200.
-        
-    Raises:
-        ValueError: If any error occurs during the chunking process.
-        
-    Returns:
-        List[Dict[str, Any]]: A list of dictionaries, each containing a separate chunk along with original key-value pairs from the input dictionary.
-    
-    Example:
-        >>> d = {'content': 'This is a test string.', 'other_field': 1}
-        >>> file_to_chunks(d)
-        [{'chunk_overlap': 0.2, 'chunk_threshold': 200, 'file_chunks': 2, 'chunk_id': 1, 'chunk_size': 14, 'chunk_content': 'This is a test', 'other_field': 1}, ...]
-    """
-
-    try:
-        out = {key: value for key, value in input.items() if key != field}
-        out.update({"chunk_overlap": overlap, "chunk_threshold": threshold})
-
-        chunks = chunk_text(input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold)
-        logs = []
-        for i, chunk in enumerate(chunks):
-            chunk_dict = out.copy()
-            chunk_dict.update({
-                'file_chunks': len(chunks),
-                'chunk_id': i + 1,
-                'chunk_size': len(chunk),
-                f'chunk_{field}': chunk
-            })
-            logs.append(chunk_dict)
-
-        return logs
-
-    except Exception as e:
-        raise ValueError(f"An error occurred while chunking the file. {e}")
-
-def file_to_chunks(input,
-                   field: str = 'content',
-                   chunk_size: int = 1500,
-                   overlap: float = 0.2,
-                   threshold: int = 200,
-                   to_csv=False,
-                   project='project',
-                   output_dir='data/logs/sources/',
-                   chunk_func = _file_to_chunks,
-                   filename=None,
-                   verbose=True,
-                   timestamp=True,
-                   logger=None):
-    """
-        Splits text from a specified dictionary field into chunks and returns a list of dictionaries.
-
-    Parameters:
-        input (List[Dict[str, Any]]): The input dictionaries containing the text field to be chunked.
-
-        field (str, optional): The dictionary key corresponding to the text field. Defaults to 'content'.
-
-        chunk_size (int, optional): Size of each text chunk in characters. Defaults to 1500.
-
-        overlap (float, optional): Percentage of overlap between adjacent chunks, in the range [0, 1]. Defaults to 0.2.
-
-        threshold (int, optional): Minimum size for the last chunk. If smaller, it will be merged with the previous chunk. Defaults to 200.
-
-        to_csv (bool, optional): If True, export the processed data to a CSV file.
-
-        project (str, optional): The name of the project.
-
-        output_dir (str, optional): The directory path for exporting the CSV file.
-
-        chunk_func (function, optional): The function to be used for chunking. Defaults to _file_to_chunks.
-
-        filename (str, optional): The name of the CSV file.
-
-        verbose (bool, optional): If True, print a verbose message after export.
-
-        timestamp (bool, optional): If True, include a timestamp in the exported file name.
-
-        logger (DataLogger, optional): An optional DataLogger instance for logging.
-
-    Returns:
-        List[Dict[str, Any]]: A list of dictionaries representing the processed text chunks.
-    """
-
-    _f = lambda x: chunk_func(x, field=field, chunk_size=chunk_size, overlap=overlap, threshold=threshold)
-    logs = to_list(l_call(input, _f), flat=True)
-
-    if to_csv:
-        filename = filename if filename else f"{project}_sources.csv"
-        logger = DataLogger(log=logs) if not logger else logger
-        logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
-
-    return logs
-
-def get_bins(input: List[str], upper: int = 7500) -> List[List[int]]:
-    """
-    Get index of elements in a list based on their consecutive cumulative sum of length,
-    according to some upper threshold. Return lists of indices as bins.
-    
-    Parameters:
-        input (List[str]): List of items to be binned.
-
-        upper (int, optional): Upper threshold for the cumulative sum of the length of items in a bin. Default is 7500.
-    
-    Returns:
-        List[List[int]]: List of lists, where each inner list contains the indices of the items that form a bin.
-    
-    Example:
-        >>> items = ['apple', 'a', 'b', 'banana', 'cheery', 'c', 'd', 'e']
-        >>> upper = 10
-        >>> get_bins(items, upper)
-        [[0, 1, 2], [3], [4, 5, 6, 7]]
-    """
-    current = 0
-    bins = []
-    bin = []
-    for idx, item in enumerate(input):
-        if current + len(item) < upper:
-            bin.append(idx)
-            current += len(item)
-        elif current + len(item) >= upper:
-            bins.append(bin)
-            bin = [idx]
-            current = len(item)
-        if idx == len(input) - 1 and len(bin) > 0:
-            bins.append(bin)
-    return bins

lionagi/utils/log_util.py

@@ -1,86 +0,0 @@
-from collections import deque
-from .sys_util import to_csv, create_path
-
-
-class DataLogger:
-    """
-    Logs data entries and outputs them to a CSV file.
-    
-    This class provides a logging mechanism for data entries that can be saved to a
-    CSV file. It offers methods for appending new log entries, saving the log to a CSV file,
-    and setting the directory where the logs should be saved.
-
-    Attributes:
-        dir (str):
-            The directory where the log files are to be saved.
-        log (deque):
-            A deque that stores log entries.
-
-    Methods:
-        __call__(entry):
-            Appends a new entry to the log.
-        to_csv(dir: str, filename: str, verbose: bool, timestamp: bool, dir_exist_ok: bool, file_exist_ok: bool): 
-            Converts the log to a CSV format and saves it to a file.
-        set_dir(dir: str):
-            Sets the directory for saving log files.
-    """
-
-    def __init__(self, dir= None, log: list = None) -> None:
-        """
-        Initializes a new instance of the DataLogger class.
-
-        Parameters:
-            dir (str, optional): The directory where the log files will be saved. Defaults to None.
-
-            log (list, optional): An initial list of log entries. Defaults to an empty deque.
-        """
-        self.dir = dir
-        self.log = deque(log) if log else deque()
-
-    def __call__(self, entry):
-        """
-        Appends a new entry to the log.
-
-        Parameters:
-            entry: The entry to append to the log. The entry can be of any datatype.
-        """
-        self.log.append(entry)
-
-    def to_csv(self, dir: str, filename: str, verbose: bool = True, timestamp: bool = True, dir_exist_ok=True, file_exist_ok=False):
-        """
-        Converts the log to a CSV format and saves it to a file.
-
-        Parameters:
-            dir (str): The directory where the CSV file will be saved.
-
-            filename (str): The name of the CSV file.
-
-            verbose (bool, optional): If True, prints a message after saving the log. Defaults to True.
-
-            timestamp (bool, optional): If True, appends a timestamp to the filename. Defaults to True.
-
-            dir_exist_ok (bool, optional): If True, overrides the existing directory if needed. Defaults to True.
-
-            file_exist_ok (bool, optional): If True, overrides the existing file if needed. Defaults to False.
-
-        Postconditions:
-            Saves the log entries to a CSV file and clears the `log` attribute.
-
-            Optionally prints a message with the number of log entries saved and the file path.
-        """
-        filepath = create_path(dir=dir, filename=filename, timestamp=timestamp, dir_exist_ok=dir_exist_ok)
-        to_csv(list(self.log), filepath, file_exist_ok=file_exist_ok)
-        n_logs = len(list(self.log))
-        self.log = deque()
-        if verbose:
-            print(f"{n_logs} logs saved to {filepath}")
-            
-    def set_dir(self, dir: str):
-        """
-        Sets the directory where log files will be saved.
-
-        Parameters:
-            dir (str): The directory to set for saving log files.
-        """
-        self.dir = dir
-        

lionagi-0.0.111.dist-info/RECORD

@@ -1,20 +0,0 @@
-lionagi/__init__.py,sha256=2Rko3tw94ZFVN_GSvcxAY1O77FxswcaMxNHKH5Bj7jc,788
-lionagi/version.py,sha256=BtvlzlO7UNG5nD9ftqoP0J2wHDhVCXpzuMsjYBw0Wfg,24
-lionagi/api/__init__.py,sha256=PF-fWsB0axACGDbm8FluZgDQMyQf3PUUJ1tIXW535TQ,178
-lionagi/api/oai_config.py,sha256=yhyZ4aEaF6r3XBbhxI47r8CL2-amc-4IKJhbXv2W9CM,356
-lionagi/api/oai_service.py,sha256=9FBmw_UTBsgnPBP7f8KOX4PwS5TXeknnUw6-kAYJxqk,11919
-lionagi/session/__init__.py,sha256=qAf0IAA2D1ZhwvRopSgi8X13DH4Y0E5HaA2DVr6FxG0,152
-lionagi/session/conversation.py,sha256=5EePoSVIcyaZJEtkNxv1yCFby_3_SLx5AmaSxS9pwI8,4058
-lionagi/session/message.py,sha256=L4QhTvay-xqL_PWWzr_lTig70cr7zEc5YjONILmAoWU,6504
-lionagi/session/session.py,sha256=wmAwNxk6pm1MTV3kqjG8i5a89xssDP0Ln8m2_e_zdek,14748
-lionagi/utils/__init__.py,sha256=e5aEzyHofUYZ8olcyHxq7wqTGRsRZwag3vyZ0T4_ByQ,842
-lionagi/utils/api_util.py,sha256=xPWwQkWMnkGOUAKpwOkGkMv2SWCSCHjBsggnrSfHJhs,15257
-lionagi/utils/doc_util.py,sha256=ZEuLKzc3EH692FW1LXRXBHgextMfb1OaLE3z_NhBBT0,12882
-lionagi/utils/log_util.py,sha256=mfLmvjv4hvTYMel46tpKJyqLbj1PZimCgKigz48osZY,3158
-lionagi/utils/sys_util.py,sha256=q4I_d61Zwe-WvukNoa53Gd8ycDcTpfOhw3yER8ZoiCg,28449
-lionagi/utils/tool_util.py,sha256=0mWGW_rfUPTay_L05dckGzEXdg4ZdhFyGA1lve9tnj8,7410
-lionagi-0.0.111.dist-info/LICENSE,sha256=TBnSyG8fs_tMRtK805GzA1cIyExleKyzoN_kuVxT9IY,11358
-lionagi-0.0.111.dist-info/METADATA,sha256=drb1EbWKei2M7WjH6Md8Qq_vOMRc8Sgz1oyxawPZQEY,17414
-lionagi-0.0.111.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-lionagi-0.0.111.dist-info/top_level.txt,sha256=szvch_d2jE1Lu9ZIKsl26Ll6BGfYfbOgt5lm-UpFSo4,8
-lionagi-0.0.111.dist-info/RECORD,,