lionagi 0.0.115__py3-none-any.whl → 0.0.204__py3-none-any.whl

lionagi/utils/pd_util.py

@@ -0,0 +1,57 @@
+from typing import List, Dict
+import pandas as pd
+from ..schema.data_node import DataNode
+from .sys_util import timestamp_to_datetime
+
+def to_pd_df(items: List[Dict], how: str = 'all') -> pd.DataFrame:
+    """
+    Converts a list of dictionaries to a pandas DataFrame, dropping NA values.
+
+    Args:
+        items (List[Dict]): A list of dictionaries to be converted to a DataFrame.
+        how (str): How to handle NA values. Options are 'all' to drop rows with all NA values,
+                   'any' to drop rows with any NA values, or 'none' to keep all rows.
+
+    Returns:
+        pd.DataFrame: A pandas DataFrame containing the data from the input list of dictionaries.
+    """
+    df = pd.DataFrame(items).dropna(how=how)
+    df.reset_index(drop=True, inplace=True)
+    return df
+
+def pd_row_to_node(row: pd.Series):
+    """
+    Converts a pandas Series row to a DataNode object with structured data.
+
+    Args:
+        row (pd.Series): A pandas Series containing row data to be converted.
+
+    Returns:
+        DataNode: A DataNode object containing structured data from the input pandas Series.
+    """
+    dict_ = row.to_dict()
+    dict_['datetime'] = str(timestamp_to_datetime(dict_['datetime']))
+    dict_['content'] = {'headline': dict_.pop('headline'), 'summary': dict_.pop('summary')}
+    dict_['metadata'] = {'datetime': dict_.pop('datetime'), 'url': dict_.pop('url'), 'id': dict_.pop('id')}
+    return DataNode.from_dict(dict_)
+
+def expand_df_datetime(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Expands a pandas DataFrame with a datetime column into separate year, month, and day columns.
+
+    Args:
+        df (pd.DataFrame): The pandas DataFrame containing a 'datetime' column to be expanded.
+
+    Returns:
+        pd.DataFrame: A pandas DataFrame with 'year', 'month', and 'day' columns.
+    """
+    df_expanded = df.copy()
+    if df_expanded['datetime'].dtype == int:
+        df_expanded['datetime'] = df_expanded['datetime'].apply(lambda x: timestamp_to_datetime(x))
+
+    df_expanded.insert(0, 'year', df_expanded['datetime'].dt.year)
+    df_expanded.insert(1, 'month', df_expanded['datetime'].dt.month)
+    df_expanded.insert(2, 'day', df_expanded['datetime'].dt.day)
+    df_expanded.drop('datetime', axis=1, inplace=True)
+
+    return df_expanded

lionagi/utils/sys_util.py

@@ -1,226 +1,354 @@
-"""   
-Copyright 2023 HaiyangLi <ocean@lionagi.ai>
+import os
+import copy
+from datetime import datetime
+import hashlib
+import re
 
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
+import json
+import logging
 
-       http://www.apache.org/licenses/LICENSE-2.0
+from typing import Any, List, Dict, Union
 
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-"""
 
-# this module has no internal dependency 
-import os
-import copy
-import hashlib
-from pathlib import Path
-from datetime import datetime
-from typing import Any, Generator, List, Dict
+def get_timestamp() -> str:
+    """
+    Generates a current timestamp in ISO format with colons and periods replaced by 
+    underscores.
+
+    Returns:
+        str: The current timestamp.
 
-def create_copy(input: Any, n: int) -> Any:
+    Examples:
+    >>> isinstance(get_timestamp(), str)
+    True
     """
-    Creates a deep copy of the input object a specified number of times.
+    return datetime.now().isoformat().replace(":", "_").replace(".", "_")
 
-    This function makes deep copies of the provided input. If the number of copies ('n') 
-    is greater than 1, a list of deep copies is returned. For a single copy, it returns 
-    the copy directly.
+def create_copy(input: Any, n: int = 1) -> Any:
+    """
+    Creates a deep copy of the given input. If 'n' is greater than 1, returns a 
+    list of deep copies.
 
-    Parameters:
+    Args:
         input (Any): The object to be copied.
+        n (int, optional): The number of copies to create. Defaults to 1.
 
-        n (int): The number of deep copies to create.
+    Returns:
+        Any: A deep copy of the input, or a list of deep copies.
 
     Raises:
         ValueError: If 'n' is not a positive integer.
-
-    Returns:
-        Any: A deep copy of 'input' or a list of deep copies if 'n' > 1.
-
-    Example:
-        >>> sample_dict = {'key': 'value'}
-        >>> make_copy(sample_dict, 2)
-        [{'key': 'value'}, {'key': 'value'}]
+        
+    Examples:
+    >>> create_copy([1, 2, 3], 2)
+    [[1, 2, 3], [1, 2, 3]]
+    >>> create_copy("Hello")
+    'Hello'
     """
     if not isinstance(n, int) or n < 1:
         raise ValueError(f"'n' must be a positive integer: {n}")
     return copy.deepcopy(input) if n == 1 else [copy.deepcopy(input) for _ in range(n)]
 
-def create_id(n=32) -> str:
+def create_path(
+    dir: str, filename: str, timestamp: bool = True, dir_exist_ok: bool = True, 
+    time_prefix: bool = False
+) -> str:
     """
-    Generates a unique ID based on the current time and random bytes.
+    Creates a file path with an optional timestamp, ensuring the directory exists.
 
-    This function combines the current time in ISO 8601 format with 16 random bytes
-    to create a unique identifier. The result is hashed using SHA-256 and the first
-    16 characters of the hexadecimal digest are returned.
+    Args:
+        dir (str): The directory in which the file will be placed.
+        filename (str): The name of the file.
+        timestamp (bool, optional): Flag to include a timestamp in the filename. Defaults to True.
+        dir_exist_ok (bool, optional): Flag to create the directory if it doesn't exist. Defaults to True.
+        time_prefix (bool, optional): Flag to place the timestamp as a prefix. Defaults to False.
 
     Returns:
-        str: A 16-character unique identifier.
+        str: The full path of the file with the optional timestamp.
 
-    Example:
-        >>> create_id()  # Doctest: +ELLIPSIS
-        '...'
+    Examples:
+    >>> create_path("/tmp", "log.txt", timestamp=False)
+    '/tmp/log.txt'
+    >>> isinstance(create_path("/tmp", "report", time_prefix=True), str)
+    True
     """
-    current_time = datetime.now().isoformat().encode('utf-8')
-    random_bytes = os.urandom(2048)
-    return hashlib.sha256(current_time + random_bytes).hexdigest()[:n]
+    dir = dir + '/' if not dir.endswith('/') else dir
+    if '.' in filename:
+        name, ext = filename.rsplit('.', 1)
+    else:
+        name, ext = filename, ''
+    os.makedirs(dir, exist_ok=dir_exist_ok)
+    timestamp_str = get_timestamp() if timestamp else ''
+    filename = f"{timestamp_str}_{name}" if time_prefix else f"{name}_{timestamp_str}"
+    return f"{dir}{filename}.{ext}" if ext else f"{dir}{filename}"
 
-def get_timestamp() -> str:
+def split_path(path: str) -> tuple:
     """
-    Generates a current timestamp in a file-safe string format.
+    Splits a file path into its directory name and file name.
 
-    This function creates a timestamp from the current time, formatted in ISO 8601 format, 
-    and replaces characters that are typically problematic in filenames (like colons and periods) 
-    with underscores.
+    Args:
+        path (str): The file path to split.
 
     Returns:
-        str: The current timestamp in a file-safe string format.
+        tuple: A tuple containing the directory name and file name.
 
-    Example:
-        >>> get_timestamp()  # Doctest: +ELLIPSIS
-        '...'
+    Examples:
+    >>> split_path("/home/user/document.txt")
+    ('/home/user', 'document.txt')
+    >>> split_path("document.txt")
+    ('', 'document.txt')
     """
-    return datetime.now().isoformat().replace(":", "_").replace(".", "_")
+    folder_name = os.path.dirname(path)
+    file_name = os.path.basename(path)
+    return (folder_name, file_name)
 
-def create_path(dir: str, filename: str, timestamp: bool = True, dir_exist_ok: bool = True, time_prefix=False) -> str:
+def str_to_num(
+    input: str, upper_bound: float = None, lower_bound: float = None, 
+    num_type: type = int, precision: int = None
+) -> Any:
     """
-    Creates a file path by optionally appending a timestamp to the filename.
-
-    This function constructs a file path by combining a directory, an optional timestamp,
-    and a filename. It also ensures the existence of the directory.
-
-    Parameters:
-        dir (str): The directory in which the file is to be located.
-
-        filename (str): The name of the file.
-
-        timestamp (bool, optional): If True, appends a timestamp to the filename. Defaults to True.
+    Converts the first number in a string to a specified numeric type and checks if it 
+    falls within specified bounds.
 
-        dir_exist_ok (bool, optional): If True, creates the directory if it doesn't exist. Defaults to True.
-
-        time_prefix (bool, optional): If True, the timestamp is added as a prefix; otherwise, it's appended. Defaults to False.
+    Args:
+        input (str): The string containing the number.
+        upper_bound (float, optional): The upper limit for the number. Defaults to None.
+        lower_bound (float, optional): The lower limit for the number. Defaults to None.
+        num_type (type): The numeric type to which the number will be converted. Default is int.
+        precision (int, optional): The precision for floating point numbers. Defaults to None.
 
     Returns:
-        str: The full path to the file.
+        Any: The converted number in the specified type.
 
-    Example:
-        >>> create_path('/tmp/', 'log.txt', timestamp=False)
-        '/tmp/log.txt'
+    Raises:
+        ValueError: If no numeric value is found, or the number is outside the specified bounds.
+
+    Examples:
+    >>> str_to_num("Value is 20.5", upper_bound=30, num_type=float)
+    20.5
+    >>> str_to_num("Temperature -5 degrees", lower_bound=0)
+    ValueError: Number -5 is less than the lower bound of 0.
     """
+    number_str = _extract_first_number(input)
+    if number_str is None:
+        raise ValueError(f"No numeric values found in the string: {input}")
     
-    dir = dir + '/' if str(dir)[-1] != '/' else dir
-    filename, ext = filename.split('.')
-    os.makedirs(dir, exist_ok=dir_exist_ok)
-    
-    if timestamp:
-        timestamp = get_timestamp()
-        return f"{dir}{timestamp}_{filename}.{ext}" if time_prefix else f"{dir}{filename}_{timestamp}.{ext}"
-    else:
-        return f"{dir}{filename}"
+    number = _convert_to_num(number_str, num_type, precision)
 
-def split_path(path: Path) -> tuple:
-    folder_name = path.parent.name
-    file_name = path.name
-    return (folder_name, file_name)
+    if upper_bound is not None and number > upper_bound:
+        raise ValueError(f"Number {number} is greater than the upper bound of {upper_bound}.")
+    if lower_bound is not None and number < lower_bound:
+        raise ValueError(f"Number {number} is less than the lower bound of {lower_bound}.")
+
+    return number
 
-def get_bins(input: List[str], upper: int = 7500) -> List[List[int]]:
+def get_bins(input: List[str], upper: int) -> List[List[int]]:
     """
-    Get index of elements in a list based on their consecutive cumulative sum of length,
-    according to some upper threshold. Return lists of indices as bins.
-    
-    Parameters:
-        input (List[str]): List of items to be binned.
+    Distributes a list of strings into bins where the total length of strings in each 
+    bin is less than a specified upper limit.
+
+    Args:
+        input (List[str]): The list of strings to be distributed.
+        upper (int): The upper limit for the total length of strings in each bin.
 
-        upper (int, optional): Upper threshold for the cumulative sum of the length of items in a bin. Default is 7500.
-    
     Returns:
-        List[List[int]]: List of lists, where each inner list contains the indices of the items that form a bin.
-    
-    Example:
-        >>> items = ['apple', 'a', 'b', 'banana', 'cheery', 'c', 'd', 'e']
-        >>> upper = 10
-        >>> get_bins(items, upper)
-        [[0, 1, 2], [3], [4, 5, 6, 7]]
+        List[List[int]]: A list of lists, where each inner list contains indices of the 
+        input list that make up a bin.
+
+    Examples:
+    >>> get_bins(["one", "two", "three", "four", "five"], 10)
+    [[0, 1], [2], [3, 4]]
     """
     current = 0
     bins = []
-    bin = []
-    
+    current_bin = []
     for idx, item in enumerate(input):
         
         if current + len(item) < upper:
-            bin.append(idx)
+            current_bin.append(idx)
             current += len(item)
-    
-        elif current + len(item) >= upper:
-            bins.append(bin)
-            bin = [idx]
+            
+        else:
+            bins.append(current_bin)
+            current_bin = [idx]
             current = len(item)
-    
-        if idx == len(input) - 1 and len(bin) > 0:
-            bins.append(bin)
-    
+            
+    if current_bin:
+        bins.append(current_bin)
+        
     return bins
 
-def change_dict_key(dict_, old_key, new_key):
-    dict_[new_key] = dict_.pop(old_key)
+def create_id(n: int = 32) -> str:
+    """
+    Creates a unique identifier using a combination of the current time and random bytes.
 
-def timestamp_to_datetime(timestamp: int) -> str:
-    if isinstance(timestamp, str):
-        try:
-            timestamp = int(timestamp)
-        except:
-            return timestamp
-    return datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
+    Args:
+        n (int, optional): The length of the identifier. Defaults to 32.
 
-def is_schema(dict_: Dict, schema: Dict):
-    for key, expected_type in schema.items():
-        if not isinstance(dict_[key], expected_type):
-            return False
-    return True
+    Returns:
+        str: The generated unique identifier.
 
-def create_hash(data: str, algorithm: str = 'sha256') -> str:
+    Examples:
+    >>> len(create_id())
+    32
+    >>> len(create_id(16))
+    16
+    """
+    current_time = datetime.now().isoformat().encode('utf-8')
+    random_bytes = os.urandom(32)
+    return hashlib.sha256(current_time + random_bytes).hexdigest()[:n]
+
+def directory_cleaner(directory: str) -> None:
+    """
+    Deletes all files within a given directory.
+
+    Args:
+        directory (str): The path of the directory to clean.
+
+    Raises:
+        FileNotFoundError: If the specified directory does not exist.
+        Exception: If a file in the directory cannot be deleted.
+
+    Examples:
+    >>> directory_cleaner("/path/to/nonexistent/directory")
+    FileNotFoundError: The specified directory does not exist.
     """
-    Generates a hash for the given data using the specified algorithm.
+    if not os.path.exists(directory):
+        raise FileNotFoundError("The specified directory does not exist.")
+    
+    for filename in os.listdir(directory):
+        file_path = os.path.join(directory, filename)
+        try:
+            if os.path.isfile(file_path) or os.path.islink(file_path):
+                os.unlink(file_path)
+                logging.info(f'Successfully deleted {file_path}')
+        except Exception as e:
+            logging.error(f'Failed to delete {file_path}. Reason: {e}')
+            raise
+
+def strip_lower(input: Any) -> str:
+    """
+    Strips and converts a given input to lower case.
 
-    Parameters:
-        data (str): The data to be hashed.
-        algorithm (str): The hashing algorithm to use (e.g., 'sha256', 'md5').
+    Args:
+        input (Any): The input to be processed.
 
     Returns:
-        str: The generated hash string.
+        str: The processed string in lower case.
+
+    Raises:
+        Exception: If the input cannot be converted to a string.
+
+    Examples:
+    >>> strip_lower(" Hello WORLD ")
+    'hello world'
+    >>> strip_lower(123)
+    '123'
     """
-    hasher = hashlib.new(algorithm)
-    hasher.update(data.encode())
-    return hasher.hexdigest()
+    try:
+        return str(input).strip().lower()
+    except:
+        return False
 
+def _extract_first_number(inputstr: str) -> str:
+    """
+    Extracts the first number from a given string.
 
+    Args:
+        inputstr (str): The string from which the number will be extracted.
 
-# def parse_function_call(response: str) -> Tuple[str, Dict]:
-#     out = json.loads(response)
-#     func = out.get('function', '').lstrip('call_')
-#     args = json.loads(out.get('arguments', '{}'))
-#     return func, args
+    Returns:
+        str: The first number found in the string, returned as a string. 
+             Returns None if no number is found.
+
+    Examples:
+    >>> extract_first_number("The 2 little pigs")
+    '2'
+    >>> extract_first_number("No numbers")
+    None
+    """
+    numbers = re.findall(r'-?\d+\.?\d*', inputstr)
+    return numbers[0] if numbers else None
 
-# ------------------------------------------------------------------------
-# credit to OpenAI for the following functions
-def task_id_generator() -> Generator[int, None, None]:
+def _convert_to_num(number_str: str, num_type: type = int, precision: int = None) -> Any:
     """
-    A generator function that yields a sequential series of task IDs.
+    Converts a string representation of a number to a specified numeric type.
+
+    Args:
+        number_str (str): The string representation of the number.
+        num_type (type): The type to which the number will be converted. Default is int.
+        precision (int, optional): The precision for floating point numbers. Defaults to None.
+
+    Returns:
+        Any: The converted number in the specified type.
 
-    Yields:
-        int: The next task ID in the sequence, starting from 0.
+    Raises:
+        ValueError: If an invalid number type is provided.
 
     Examples:
-        task_id_gen = task_id_generator()
-        next(task_id_gen) # Yields 0
-        next(task_id_gen) # Yields 1
-    """
-    task_id = 0
-    while True:
-        yield task_id
-        task_id += 1
+    >>> convert_to_num("3.142", float, 2)
+    3.14
+    >>> convert_to_num("100", int)
+    100
+    """
+    if num_type is int:
+        return int(float(number_str))
+    elif num_type is float:
+        return round(float(number_str), precision) if precision is not None else float(number_str)
+    else:
+        raise ValueError(f"Invalid number type: {num_type}")
+
+def change_dict_key(dict_: Dict[Any, Any], old_key: str, new_key: str) -> None:
+    """
+    Changes a key in a dictionary to a new key.
+
+    Args:
+        dict_ (Dict[Any, Any]): The dictionary to change the key in.
+        old_key (str): The old key to be changed.
+        new_key (str): The new key to change to.
+    """
+    if old_key in dict_:
+        dict_[new_key] = dict_.pop(old_key)
+
+def as_dict(input_: Any) -> Dict[Any, Any]:
+    """
+    Converts a JSON string or a dictionary to a dictionary.
+
+    Args:
+        input_: A JSON string or dictionary to convert.
+
+    Returns:
+        The input converted to a dictionary.
+
+    Raises:
+        ValueError: If the input cannot be converted to a dictionary.
+    """
+    if isinstance(input_, str):
+        try:
+            return json.loads(input_)
+        except Exception as e:
+            raise f"Could not convert input to dict: {e}"
+    elif isinstance(input_, dict):
+        return input_
+    else:
+        raise f"Could not convert input to dict: {input_}"
+
+def is_schema(dict_: Dict, schema: Dict):
+    """
+    Checks if a dictionary matches a given schema.
+
+    Args:
+        dict_: The dictionary to check.
+        schema: The schema to validate against.
+
+    Returns:
+        True if the dictionary matches the schema, False otherwise.
+    """
+    for key, expected_type in schema.items():
+        if not isinstance(dict_[key], expected_type):
+            return False
+    return True
+
+def timestamp_to_datetime(timestamp):    
+    return datetime.fromtimestamp(timestamp)
+

lionagi/utils/url_util.py

@@ -0,0 +1,55 @@
+from bs4 import BeautifulSoup
+import requests
+
+def get_url_response(url: str, timeout: tuple = (1, 1), **kwargs) -> requests.Response:
+    """
+    Sends a GET request to a URL and returns the response.
+
+    Args:
+        url (str): The URL to send the GET request to.
+        timeout (tuple): A tuple specifying the connection and read timeouts in seconds.
+                         Defaults to (1, 1).
+        **kwargs: Additional keyword arguments to be passed to the requests.get() function.
+
+    Returns:
+        requests.Response: A Response object containing the server's response to the GET request.
+
+    Raises:
+        TimeoutError: If a timeout occurs while requesting or reading the response.
+        Exception: If an error other than a timeout occurs during the request.
+    """
+    try:
+        response = requests.get(url, timeout=timeout, **kwargs)
+        response.raise_for_status()
+        return response
+    except requests.exceptions.ConnectTimeout:
+        raise TimeoutError(f"Timeout: requesting >{timeout[0]} seconds.")
+    except requests.exceptions.ReadTimeout:
+        raise TimeoutError(f"Timeout: reading >{timeout[1]} seconds.")
+    except Exception as e:
+        raise e
+    
+def get_url_content(url: str) -> str:
+    """
+    Retrieve and parse the content from a given URL.
+
+    Args:
+        url (str): The URL to fetch and parse.
+
+    Returns:
+        str: The text content extracted from the URL.
+
+    Raises:
+        ValueError: If there is an issue during content retrieval or parsing.
+    """
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        text_content = ' '.join([p.get_text() for p in soup.find_all('p')])
+        return text_content
+    except Exception as e:
+        raise f"Error fetching content for {url}: {e}"
+    

lionagi/version.py

@@ -1 +1 @@
-__version__ = "0.0.115"
+__version__ = "0.0.204"