lionagi 0.0.112__py3-none-any.whl → 0.0.113__py3-none-any.whl

lionagi/utils/load_utils.py

@@ -0,0 +1,190 @@
+import math
+from pathlib import Path
+from typing import List, Union, Dict, Any, Tuple
+
+from .type_util import to_list
+from .call_util import lcall
+from .io_util import to_csv
+from ..schema.base_schema import DataNode
+
+
+def dir_to_path(
+    dir: str, ext: str, recursive: bool = False, 
+    flatten: bool = True) -> List[Path]:
+    """
+    Generates a list of file paths from a directory with the given file extension.
+
+    Args:
+        directory (str): The directory to search for files.
+        extension (str): The file extension to filter by.
+        recursive (bool): Whether to search subdirectories recursively. Defaults to False.
+        flatten (bool): Whether to flatten the list. Defaults to True.
+    
+    Returns:
+        List[Path]: A list of Paths to the files.
+
+    Raises:
+        ValueError: If the directory or extension is invalid.
+    """
+    
+    def _dir_to_path(ext):
+        tem = '**/*' if recursive else '*'
+        return list(Path(dir).glob(tem + ext))
+    
+    try: 
+        return to_list(lcall(ext, _dir_to_path, flatten=True), flatten=flatten)
+    except: 
+        raise ValueError("Invalid directory or extension, please check the path")
+
+def dir_to_nodes(dir: str, ext, recursive: bool = False, flatten: bool = True, clean_text: bool = True):
+    path_list = dir_to_path(dir, ext, recursive, flatten)
+    files_info = lcall(path_list, read_text, clean=clean_text)
+    nodes = lcall(files_info, lambda x: DataNode(content=x[0], metadata=x[1]))
+    return nodes
+
+def chunk_text(input: str, 
+               chunk_size: int, 
+               overlap: float,
+               threshold: int) -> List[Union[str, None]]:
+    """
+    Chunks the input text into smaller parts, with optional overlap and threshold for final chunk.
+
+    Args:
+        text (str): The input text to chunk.
+        chunk_size (int): The size of each chunk.
+        overlap (float): The amount of overlap between chunks.
+        threshold (int): The minimum size of the final chunk.
+
+    Returns:
+        List[Union[str, None]]: A list of text chunks.
+
+    Raises:
+        ValueError: If an error occurs during chunking.
+    """
+    
+    def _chunk_n1():
+        return [input]
+        
+    def _chunk_n2():
+        chunks = []
+        chunks.append(input[:chunk_size + overlap_size])
+        
+        if len(input) - chunk_size > threshold:
+            chunks.append(input[chunk_size - overlap_size:])    
+        else:
+            return _chunk_n1()
+        
+        return chunks
+
+    def _chunk_n3():
+        chunks = []
+        chunks.append(input[:chunk_size + overlap_size])
+        for i in range(1, n_chunks - 1):
+            start_idx = chunk_size * i - overlap_size
+            end_idx = chunk_size * (i + 1) + overlap_size
+            chunks.append(input[start_idx:end_idx])
+
+        if len(input) - chunk_size * (n_chunks - 1) > threshold:
+            chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size:])
+        else:
+            chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size:]
+
+        return chunks
+    
+    try:
+        if not isinstance(input, str): input = str(input)
+        
+        n_chunks = math.ceil(len(input) / chunk_size)
+        overlap_size = int(overlap / 2)
+
+        if n_chunks == 1: 
+            return _chunk_n1()
+        
+        elif n_chunks == 2:
+            return _chunk_n2()
+        
+        elif n_chunks > 2:
+            return _chunk_n3()
+
+    except Exception as e:
+        raise ValueError(f"An error occurred while chunking the text. {e}")
+
+def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
+    """
+    Reads text from a file and optionally cleans it, returning the content and metadata.
+
+    Args:
+        filepath (str): The path to the file to read.
+        clean (bool): Whether to clean the text by replacing certain characters. Defaults to True.
+
+    Returns:
+        Tuple[str, dict]: A tuple containing the content and metadata of the file.
+
+    Raises:
+        FileNotFoundError: If the file cannot be found.
+        PermissionError: If there are permissions issues.
+        OSError: For other OS-related errors.
+    """
+    def _get_metadata():
+        import os
+        from datetime import datetime
+        file = filepath
+        size = os.path.getsize(filepath)
+        creation_date = datetime.fromtimestamp(os.path.getctime(filepath)).date()
+        modified_date = datetime.fromtimestamp(os.path.getmtime(filepath)).date()
+        last_accessed_date = datetime.fromtimestamp(os.path.getatime(filepath)).date()
+        return {'file': str(file),
+                'size': size,
+                'creation_date': str(creation_date),
+                'modified_date': str(modified_date),
+                'last_accessed_date': str(last_accessed_date)}
+    try:
+        with open(filepath, 'r') as f:
+            content = f.read()
+            if clean:
+                # Define characters to replace and their replacements
+                replacements = {'\\': ' ', '\n': ' ', '\t': ' ', '  ': ' ', '\'': ' '}
+                for old, new in replacements.items():
+                    content = content.replace(old, new)
+            metadata = _get_metadata()
+            return content, metadata
+    except Exception as e:
+        raise e
+    
+def _file_to_chunks(input: Dict[str, Any],
+                   field: str = 'content',
+                   chunk_size: int = 1500,
+                   overlap: float = 0.1,
+                   threshold: int = 200) -> List[Dict[str, Any]]:
+    try:
+        out = {key: value for key, value in input.items() if key != field}
+        out.update({"chunk_overlap": overlap, "chunk_threshold": threshold})
+
+        chunks = chunk_text(input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold)
+        logs = []
+        for i, chunk in enumerate(chunks):
+            chunk_dict = out.copy()
+            chunk_dict.update({
+                'file_chunks': len(chunks),
+                'chunk_id': i + 1,
+                'chunk_size': len(chunk),
+                f'chunk_{field}': chunk
+            })
+            logs.append(chunk_dict)
+
+        return logs
+
+    except Exception as e:
+        raise ValueError(f"An error occurred while chunking the file. {e}")
+
+def file_to_chunks(input,
+                #    project='project',
+                #    output_dir='data/logs/sources/',
+                   chunk_func = _file_to_chunks,  **kwargs):
+                #    out_to_csv=False,
+                #    filename=None,
+                #    verbose=True,
+                #    timestamp=True,
+                #    logger=None,
+    logs = to_list(lcall(input, chunk_func, **kwargs), flatten=True)
+    return logs

lionagi/utils/sys_util.py

@@ -0,0 +1,191 @@
+"""   
+Copyright 2023 HaiyangLi <ocean@lionagi.ai>
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+"""
+import os
+import copy
+import hashlib
+from pathlib import Path
+from datetime import datetime
+from typing import Any, Generator, List
+
+def create_copy(input: Any, n: int) -> Any:
+    """
+    Creates a deep copy of the input object a specified number of times.
+
+    This function makes deep copies of the provided input. If the number of copies ('n') 
+    is greater than 1, a list of deep copies is returned. For a single copy, it returns 
+    the copy directly.
+
+    Parameters:
+        input (Any): The object to be copied.
+
+        n (int): The number of deep copies to create.
+
+    Raises:
+        ValueError: If 'n' is not a positive integer.
+
+    Returns:
+        Any: A deep copy of 'input' or a list of deep copies if 'n' > 1.
+
+    Example:
+        >>> sample_dict = {'key': 'value'}
+        >>> make_copy(sample_dict, 2)
+        [{'key': 'value'}, {'key': 'value'}]
+    """
+    if not isinstance(n, int) or n < 1:
+        raise ValueError(f"'n' must be a positive integer: {n}")
+    return copy.deepcopy(input) if n == 1 else [copy.deepcopy(input) for _ in range(n)]
+
+def create_id(n=32) -> str:
+    """
+    Generates a unique ID based on the current time and random bytes.
+
+    This function combines the current time in ISO 8601 format with 16 random bytes
+    to create a unique identifier. The result is hashed using SHA-256 and the first
+    16 characters of the hexadecimal digest are returned.
+
+    Returns:
+        str: A 16-character unique identifier.
+
+    Example:
+        >>> create_id()  # Doctest: +ELLIPSIS
+        '...'
+    """
+    current_time = datetime.now().isoformat().encode('utf-8')
+    random_bytes = os.urandom(2048)
+    return hashlib.sha256(current_time + random_bytes).hexdigest()[:n]
+
+def get_timestamp() -> str:
+    """
+    Generates a current timestamp in a file-safe string format.
+
+    This function creates a timestamp from the current time, formatted in ISO 8601 format, 
+    and replaces characters that are typically problematic in filenames (like colons and periods) 
+    with underscores.
+
+    Returns:
+        str: The current timestamp in a file-safe string format.
+
+    Example:
+        >>> get_timestamp()  # Doctest: +ELLIPSIS
+        '...'
+    """
+    return datetime.now().isoformat().replace(":", "_").replace(".", "_")
+
+def create_path(dir: str, filename: str, timestamp: bool = True, dir_exist_ok: bool = True, time_prefix=False) -> str:
+    """
+    Creates a file path by optionally appending a timestamp to the filename.
+
+    This function constructs a file path by combining a directory, an optional timestamp,
+    and a filename. It also ensures the existence of the directory.
+
+    Parameters:
+        dir (str): The directory in which the file is to be located.
+
+        filename (str): The name of the file.
+
+        timestamp (bool, optional): If True, appends a timestamp to the filename. Defaults to True.
+
+        dir_exist_ok (bool, optional): If True, creates the directory if it doesn't exist. Defaults to True.
+
+        time_prefix (bool, optional): If True, the timestamp is added as a prefix; otherwise, it's appended. Defaults to False.
+
+    Returns:
+        str: The full path to the file.
+
+    Example:
+        >>> create_path('/tmp/', 'log.txt', timestamp=False)
+        '/tmp/log.txt'
+    """
+    
+    dir = dir + '/' if str(dir)[-1] != '/' else dir
+    filename, ext = filename.split('.')
+    os.makedirs(dir, exist_ok=dir_exist_ok)
+    
+    if timestamp:
+        timestamp = get_timestamp()
+        return f"{dir}{timestamp}_{filename}.{ext}" if time_prefix else f"{dir}{filename}_{timestamp}.{ext}"
+    else:
+        return f"{dir}{filename}"
+
+def split_path(path: Path) -> tuple:
+    folder_name = path.parent.name
+    file_name = path.name
+    return (folder_name, file_name)
+
+def get_bins(input: List[str], upper: int = 7500) -> List[List[int]]:
+    """
+    Get index of elements in a list based on their consecutive cumulative sum of length,
+    according to some upper threshold. Return lists of indices as bins.
+    
+    Parameters:
+        input (List[str]): List of items to be binned.
+
+        upper (int, optional): Upper threshold for the cumulative sum of the length of items in a bin. Default is 7500.
+    
+    Returns:
+        List[List[int]]: List of lists, where each inner list contains the indices of the items that form a bin.
+    
+    Example:
+        >>> items = ['apple', 'a', 'b', 'banana', 'cheery', 'c', 'd', 'e']
+        >>> upper = 10
+        >>> get_bins(items, upper)
+        [[0, 1, 2], [3], [4, 5, 6, 7]]
+    """
+    current = 0
+    bins = []
+    bin = []
+    
+    for idx, item in enumerate(input):
+        
+        if current + len(item) < upper:
+            bin.append(idx)
+            current += len(item)
+    
+        elif current + len(item) >= upper:
+            bins.append(bin)
+            bin = [idx]
+            current = len(item)
+    
+        if idx == len(input) - 1 and len(bin) > 0:
+            bins.append(bin)
+    
+    return bins
+
+def task_id_generator() -> Generator[int, None, None]:
+    """
+    A generator function that yields a sequential series of task IDs.
+
+    Yields:
+        int: The next task ID in the sequence, starting from 0.
+
+    Examples:
+        task_id_gen = task_id_generator()
+        next(task_id_gen) # Yields 0
+        next(task_id_gen) # Yields 1
+    """
+    task_id = 0
+    while True:
+        yield task_id
+        task_id += 1
+
+def change_dict_key(dict_, old_key, new_key):
+    dict_[new_key] = dict_.pop(old_key)
+
+# def parse_function_call(response: str) -> Tuple[str, Dict]:
+#     out = json.loads(response)
+#     func = out.get('function', '').lstrip('call_')
+#     args = json.loads(out.get('arguments', '{}'))
+#     return func, args

lionagi/utils/tool_util.py

@@ -0,0 +1,92 @@
+import inspect
+from ..schema.base_tool import Tool
+
+
+def extract_docstring_details(func):
+    """
+    Extracts detailed descriptions for each parameter and the function from the docstring.
+
+    Args:
+    - func (function): The function to extract details from.
+
+    Returns:
+    - Tuple[str, dict]: Function description and a dictionary of parameter descriptions.
+    """
+    docstring = inspect.getdoc(func)
+    if not docstring:
+        return "No description available.", {}
+
+    # Splitting the docstring into lines
+    lines = docstring.split('\n')
+
+    # Extracting the function description
+    func_description = lines[0].strip()
+
+    # Extracting parameter descriptions
+    param_descriptions = {}
+    current_param = None
+    for line in lines[1:]:
+        line = line.strip()
+        if line.startswith(':param'):
+            _, param, desc = line.split(' ', 2)
+            current_param = param.strip(':')
+            param_descriptions[current_param] = desc
+        elif current_param and line:
+            # Continue the description of the current parameter
+            param_descriptions[current_param] += ' ' + line
+
+    return func_description, param_descriptions
+
+def func_to_schema(func):
+    """
+    Generates a schema description for a given function, using typing hints and docstrings.
+    The schema includes the function's name, description, and parameters.
+
+    Args:
+    - func (function): The function to generate a schema for.
+
+    Returns:
+    - dict: A schema describing the function.
+    """
+    # Extracting function name and docstring details
+    func_name = func.__name__
+    func_description, param_descriptions = extract_docstring_details(func)
+    
+    # Extracting parameters with typing hints
+    sig = inspect.signature(func)
+    parameters = {
+        "type": "object",
+        "properties": {},
+        "required": [],
+    }
+    
+    for name, param in sig.parameters.items():
+        # Default type to string and update if type hint is available
+        param_type = "string"
+        if param.annotation is not inspect.Parameter.empty:
+            param_type = param.annotation.__name__
+
+        # Extract parameter description from docstring, if available
+        param_description = param_descriptions.get(name, "No description available.")
+
+        # Assuming all parameters are required for simplicity
+        parameters["required"].append(name)
+        parameters["properties"][name] = {
+            "type": param_type,
+            "description": param_description,
+        }
+    
+    # Constructing the schema
+    schema = {
+        "type": "function",
+        "function": {
+            "name": func_name,
+            "description": func_description,
+            "parameters": parameters,
+        }
+    }
+    return schema
+
+def func_to_tool(func_, schema, parser=None):
+    # schema = func_to_schema(func_)
+    return Tool(func=func_, parser=parser, schema_=schema)

lionagi/utils/type_util.py

@@ -0,0 +1,81 @@
+import re
+from typing import Optional, Union, Iterable, List, Any, Type
+
+from .flat_util import flatten_list
+
+
+def str_to_num(input_: str, 
+               upper_bound: Optional[Union[int, float]] = None, 
+               lower_bound: Optional[Union[int, float]] = None, 
+               num_type: Type[Union[int, float]] = int, 
+               precision: Optional[int] = None) -> Union[int, float]:
+    """
+    Converts the first number in the input string to the specified numeric type.
+
+    Args:
+        input_str (str): The input string to extract the number from.
+        
+        upper_bound (Optional[Union[int, float]]): The upper bound for the number. Defaults to None.
+        
+        lower_bound (Optional[Union[int, float]]): The lower bound for the number. Defaults to None.
+        
+        num_type (Type[Union[int, float]]): The type of the number to return (int or float). Defaults to int.
+        
+        precision (Optional[int]): The precision for the floating-point number. Defaults to None.
+
+    Returns:
+        Union[int, float]: The converted number.
+
+    Raises:
+        ValueError: If no numeric values are found in the string or if there are conversion errors.
+    """
+    numbers = re.findall(r'-?\d+\.?\d*', input_)
+    if not numbers:
+        raise ValueError(f"No numeric values found in the string: {input_}")
+    
+    try:
+        numbers = numbers[0]
+        if num_type is int: 
+            numbers = int(float(numbers))
+        elif num_type is float:
+            numbers = round(float(numbers), precision) if precision is not None else float(numbers)
+        else:
+            raise ValueError(f"Invalid number type: {num_type}")
+        if upper_bound is not None and numbers > upper_bound:
+            raise ValueError(f"Number {numbers} is greater than the upper bound of {upper_bound}.")
+        if lower_bound is not None and numbers < lower_bound:
+            raise ValueError(f"Number {numbers} is less than the lower bound of {lower_bound}.")
+        return numbers
+    
+    except ValueError as e:
+        raise ValueError(f"Error converting string to number: {e}")
+    
+def to_list(input_: Any, flatten: bool = True, dropna: bool = False) -> List[Any]:
+    """
+    Converts the input to a list, optionally flattening it and dropping None values.
+
+    Args:
+        input_item (Any): The input to convert to a list.
+        
+        flatten (bool): Whether to flatten the input if it is a nested list. Defaults to True.
+        
+        dropna (bool): Whether to drop None values from the list. Defaults to False.
+
+    Returns:
+        List[Any]: The input converted to a list.
+
+    Raises:
+        ValueError: If the input cannot be converted to a list.
+    """
+    if isinstance(input_, list) and flatten:
+        input_ = flatten_list(input_)
+        if dropna:
+            input_ = [i for i in input_ if i is not None]
+    elif isinstance(input_, Iterable) and not isinstance(input_, (str, dict)):
+        try:
+            input_ = list(input_)
+        except:
+            raise ValueError("Input cannot be converted to a list.")
+    else:
+        input_ = [input_]
+    return input_

lionagi/version.py

@@ -1 +1 @@
-__version__ = "0.0.112"
+__version__ = "0.0.113"

{lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lionagi
-Version: 0.0.112
+Version: 0.0.113
 Summary: Towards automated general intelligence.
 Author: HaiyangLi
 Author-email: Haiyang Li <ocean@lionagi.ai>
@@ -220,7 +220,11 @@ Requires-Dist: python-dotenv ==1.0.0
 Requires-Dist: tiktoken ==0.5.1
 Requires-Dist: httpx ==0.25.1
 
-![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935) [![Downloads](https://static.pepy.tech/badge/lionagi/week)](https://pepy.tech/project/lionagi)
+![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935)  ![Read the Docs](https://img.shields.io/readthedocs/lionagi)  ![PyPI - License](https://img.shields.io/pypi/l/lionagi?color=231fc935) ![PyPI - Downloads](https://img.shields.io/pypi/dm/lionagi?color=blue)
+
+
+
+
 
 [PyPI](https://pypi.org/project/lionagi/) | [Documentation](https://lionagi.readthedocs.io/en/latest/) | [Discord](https://discord.gg/7RGWqpSxze)
 
@@ -239,18 +243,20 @@ Install LionAGI with pip:
 ```bash
 pip install lionagi
 ```
-Download the `.env_template` file, input your OPENAI_API_KEY, save the file, rename as `.env` and put in your project's root directory. 
+Download the `.env_template` file, input your appropriate `API_KEY`, save the file, rename as `.env` and put in your project's root directory. 
+by default we use `OPENAI_API_KEY`.
 
-### Features
 
-- Robust performance
-- Efficient data operations for reading, chunking, binning, writing, storing and managing data.
-- Fast interaction with LLM services like OpenAI with **configurable rate limiting concurrent API calls** for maximum throughput. 
-- Create a production ready LLM application **in hours**. Intuitive workflow management to streamline the process from idea to market.
-- (Work In Progress): verstile intergration with most API and local LLM services.  
 
+### Features
+- Create a production ready LLM application **in hours**, with more than 100 models to choose from
+- written in pure python, minimum dependency `aiohttp`, `python-dotenv`, `tiktoken`, `pydantic`
+- Efficient and verstile data operations for reading, chunking, binning, writing, storing data with built-in support for `langchain` and `llamaindex`
+- Unified interface with any LLM provider, API or local
+  - Fast and **concurrent** API call with **configurable rate limit**
+  - (Work In Progress) support for hundreds of models both API and local
 ---
-LionAGI is designed to be async only, please check python official documentation on how `async` work: [here](https://docs.python.org/3/library/asyncio.html)
+LionAGI is designed to be `asynchronous` only, please check python official documentation on how `async` work: [here](https://docs.python.org/3/library/asyncio.html)
 
 
 **Notice**: 
@@ -271,11 +277,11 @@ import lionagi as li
 system = "You are a helpful assistant designed to perform calculations."
 instruction = {"Addition":"Add the two numbers together i.e. x+y"}
 context = {"x": 10, "y": 5}
+```
 
-# Initialize a session with a system message
+```python
+# in interactive environment (.ipynb for example)
 calculator = li.Session(system=system)
-
-# run a LLM API call
 result = await calculator.initiate(instruction=instruction,
                                    context=context,
                                    model="gpt-4-1106-preview")
@@ -283,6 +289,24 @@ result = await calculator.initiate(instruction=instruction,
 print(f"Calculation Result: {result}")
 ```
 
+```python
+# or otherwise, you can use
+import asyncio
+from dotenv import loadenv
+
+load_dotenv()
+
+async def main():
+    calculator = li.Session(system=system)
+    result = await calculator.initiate(instruction=instruction,
+                                       context=context, 
+                                       model="gpt-4-1106-preview")
+    print(f"Calculation Result: {result}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
 Visit our notebooks for our examples. 
 
 ### Community