lionagi 0.0.201__py3-none-any.whl → 0.0.204__py3-none-any.whl

lionagi/utils/io_util.py

@@ -6,6 +6,9 @@ from collections.abc import Iterable
 from typing import Any, Dict, List
 
 class IOUtil:
+    """
+    Utility class for reading and writing data from/to files in various formats.
+    """
     
     @staticmethod
     def read_csv(filepath: str) -> List[Dict[str, Any]]:

lionagi/utils/nested_util.py

@@ -1,42 +1,9 @@
 import json
 from collections import defaultdict
 from copy import deepcopy
-from itertools import chain
-from typing import Any, Callable, Dict, Generator, List, Tuple, Union, Iterable, Optional
-
-def to_list(input_: Any, flatten: bool = True, dropna: bool = False) -> List[Any]:
-    """
-    Converts the input to a list, optionally flattening it and dropping None values.
-
-    Args:
-        input_ (Any): The input to convert to a list.
-        flatten (bool): Whether to flatten the input if it is a nested list. Defaults to True.
-        dropna (bool): Whether to drop None values from the list. Defaults to False.
-
-    Returns:
-        List[Any]: The input converted to a list.
-
-    Raises:
-        ValueError: If the input cannot be converted to a list.
 
-    Examples:
-        >>> to_list([1, [2, [3, 4]], 5])
-        [1, 2, 3, 4, 5]
-        >>> to_list([1, None, 3], dropna=True)
-        [1, 3]
-    """
-    if isinstance(input_, list) and flatten:
-        input_ = _flatten_list(input_)
-    elif isinstance(input_, Iterable) and not isinstance(input_, (str, dict)):
-        try:
-            input_ = list(input_)
-        except:
-            raise ValueError("Input cannot be converted to a list.")
-    else:
-        input_ = [input_]
-    if dropna:
-        input_ = [i for i in input_ if i is not None]
-    return input_
+from typing import Any, Callable, Dict, Generator, List, Tuple, Union, Iterable, Optional
+from itertools import chain
 
 def to_readable_dict(input: Union[Dict, List]) -> Union[str, List]:
     """
@@ -433,24 +400,11 @@ def get_flattened_keys(obj: Any, sep: str = '_', max_depth: Union[int, None] = N
     else:
         return list(flatten(obj, sep=sep, max_depth=max_depth, dict_only=dict_only).keys())
 
-def _flatten_list(l: List, dropna: bool = True) -> List:
-    """
-    Flattens a nested list into a single-level list.
-
-    Args:
-        l (List): The nested list to flatten.
-        dropna (bool, optional): If True, removes None values from the flattened list. Defaults to True.
-
-    Returns:
-        List: The flattened list, with or without None values based on the dropna parameter.
-    """
-    flattened_list = list(_flatten_list_generator(l, dropna))
-    return _dropna(flattened_list) if dropna else flattened_list
-
-
-def _dynamic_flatten_in_place(obj: Any, parent_key: str = '', sep: str = '_', 
-                            max_depth: Union[int, None] = None, 
-                            current_depth: int = 0, dict_only: bool = False) -> None:
+def _dynamic_flatten_in_place(
+    obj: Any, parent_key: str = '', sep: str = '_', 
+    max_depth: Union[int, None] = None, current_depth: int = 0, 
+    dict_only: bool = False
+) -> None:
     """
     Helper function to flatten the object in place.
 
@@ -502,23 +456,10 @@ def _handle_list_insert(sub_obj: List, part: int, value: Any):
     while len(sub_obj) <= part:
         sub_obj.append(None)
     sub_obj[part] = value
-        
-def _convert_to_int_if_possible(s: str) -> Union[int, str]:
-    """
-    Converts a string to an integer if possible; otherwise, returns the string.
 
-    Args:
-        s (str): The string to convert.
-
-    Returns:
-        Union[int, str]: The converted integer or the original string.
-    """
-    return int(s) if s.lstrip('-').isdigit() else s
-
-def _is_iterable(obj) -> bool:
-    return isinstance(obj, Iterable) and not isinstance(obj, str)
-
-def _ensure_list_index(lst: List, index: int, default=None) -> None:
+def _ensure_list_index(
+    lst: List, index: int, default=None
+) -> None:
     """Ensures that a list is at least as long as a specified index.
 
     This method appends the default value to the list until its length is at least
@@ -614,52 +555,10 @@ def _deep_merge_dicts(dict1: Dict, dict2: Dict) -> Dict:
         else:
             dict1[key] = dict2[key]
     return dict1
-
-def _extend_list_to_index(lst: List[Any], index: int, fill_value: Any = None) -> None:
-    """
-    Extends a list to a specified index with a fill value.
-
-    Args:
-        lst (List[Any]): The list to extend.
-        index (int): The index to extend the list to.
-        fill_value (Any, optional): The value to fill the extended part of the list with. Defaults to None.
-
-    Returns:
-        None: This function modifies the input list in place and returns None.
-    """
-    while len(lst) <= index:
-        lst.append(fill_value)
-            
-def _dropna(l: List) -> List:
-    """
-    Removes None values from a list.
-
-    Args:
-        l (List): The list to remove None values from.
-
-    Returns:
-        List: A new list with None values removed.
-    """
-    return [item for item in l if item is not None]
-
-def _flatten_list_generator(l: List, dropna: bool = True) -> Generator[Any, None, None]:
-    """
-    Generator to flatten a nested list.
-
-    Args:
-        l (List): The nested list to flatten.
-        dropna (bool, optional): If True, removes None values during the flattening process. Defaults to True.
-
-    Yields:
-        Generator[Any, None, None]: Yields elements of the flattened list.
-    """
-    for i in l:
-        if isinstance(i, list):
-            yield from _flatten_list_generator(i, dropna)
-        else:
-            yield i
-            
-def _is_homogeneous(iterables: List[Union[Dict, List, Iterable]], type_check: type) -> bool:
+   
+def _is_homogeneous(
+    iterables: List[Union[Dict, List, Iterable]], type_check: type
+) -> bool:
     """Checks if all elements in a list are of the same specified type.
 
     Args:
@@ -753,7 +652,9 @@ def _filter_list(lst: List[Any], condition: Callable[[Any], bool]) -> List[Any]:
     """
     return [item for item in lst if condition(item)]
 
-def _get_target_container(nested_list: Union[List, Dict], indices: List[Union[int, str]]) -> Union[List, Dict]:
+def _get_target_container(
+    nested_list: Union[List, Dict], indices: List[Union[int, str]]
+) -> Union[List, Dict]:
     current_element = nested_list
     for index in indices:
         if isinstance(current_element, list):
@@ -769,98 +670,3 @@ def _get_target_container(nested_list: Union[List, Dict], indices: List[Union[in
         else:
             raise TypeError("Current element is neither a list nor a dictionary")
     return current_element
-
-
-# def nprint(nested_structure: Union[Dict, Iterable], indent: int = 0) -> None:
-#     if isinstance(nested_structure, dict):
-#         nprint(_print_dict(nested_structure, indent))
-#     elif _is_iterable(nested_structure):
-#         nprint(" " * indent + "[")
-#         nprint(_print_iterable(nested_structure, indent + 2))
-#         nprint(" " * indent + "]")
-#     else:
-#         nprint(f"{' ' * indent}{repr(nested_structure)}")
-
-# def _print_dict(nested_dict: Dict, indent: int) -> str:
-#     lines = [" " * indent + "{"]
-#     items = list(nested_dict.items())
-#     for index, (key, value) in enumerate(items):
-#         line_end = "," if index < len(items) - 1 else ""
-#         if isinstance(value, dict):
-#             lines.append(f"{' ' * (indent + 2)}{repr(key)}: {{")
-#             lines.append(_print_dict(value, indent + 4))
-#             lines.append(f"{' ' * (indent + 2)}}}{line_end}")
-#         elif _is_iterable(value):
-#             lines.append(f"{' ' * (indent + 2)}{repr(key)}: [")
-#             lines.append(_print_iterable(value, indent + 4))
-#             lines.append(f"{' ' * (indent + 2)}]{line_end}")
-#         else:
-#             lines.append(f"{' ' * (indent + 2)}{repr(key)}: {repr(value)}{line_end}")
-#     lines.append(" " * indent + "}")
-#     return "\n".join(lines)
-
-# @staticmethod
-# def _print_iterable(iterable: Iterable, indent: int) -> str:
-#     lines = []
-#     iterable_list = list(iterable)
-#     for index, item in enumerate(iterable_list):
-#         line_end = "," if index < len(iterable_list) - 1 else ""
-#         if isinstance(item, dict):
-#             lines.append(_print_dict(item, indent + 2) + line_end)
-#         elif _is_iterable(item):
-#             lines.append(" " * indent + "[")
-#             lines.append(_print_iterable(item, indent + 2))
-#             lines.append(f"{' ' * indent}]{line_end}")
-#         else:
-#             lines.append(f"{' ' * indent}{repr(item)}{line_end}")
-#     return "\n".join(lines)
-
-# def _tuples_to_dict(tuples_list: List[Tuple[str, Any]]) -> Dict:
-#     """Converts a list of tuples into a dictionary.
-
-#     Each tuple in the list should be a key-value pair, where the key is a string
-#     that represents the nested keys separated by underscores.
-
-#     Args:
-#         tuples_list: A list of tuples where each tuple is a (key, value) pair.
-
-#     Returns:
-#         A dictionary created from the tuples.
-#     """
-#     result_dict = {}
-#     for key, value in tuples_list:
-#         ninsert(result_dict, key.split('_'), value)
-#     return result_dict
-
-
-# @staticmethod
-# def _insert_with_dict_handling(result_list: Union[Dict, List], 
-#                             indices: List[Union[int, str]], 
-#                             value: Any, 
-#                             current_depth: int = 0):
-#     """
-#     Helper function to insert a value into a list or dictionary at a nested location 
-#     defined by indices.
-
-#     Args:
-#         result_list (Union[Dict, List]): The list or dictionary to insert into.
-#         indices (List[Union[int, str]]): The indices defining where to insert the value.
-#         value (Any): The value to insert.
-#         current_depth (int, optional): The current depth in the nested structure. Defaults to 0.
-
-#     Returns:
-#         None: This function modifies the input list or dictionary in place and returns None.
-#     """
-#     for index in indices[:-1]:
-#         if isinstance(result_list, list):
-#             _extend_list_to_index(result_list, index, [])
-#             result_list = result_list[index]
-#         elif isinstance(result_list, dict):
-#             result_list = result_list.setdefault(index, {})
-#         current_depth += 1
-#     last_index = indices[-1]
-#     if isinstance(result_list, list):
-#         _extend_list_to_index(result_list, last_index)
-#         result_list[last_index] = value
-#     else:
-#         result_list[last_index] = value

lionagi/utils/pd_util.py

@@ -0,0 +1,57 @@
+from typing import List, Dict
+import pandas as pd
+from ..schema.data_node import DataNode
+from .sys_util import timestamp_to_datetime
+
+def to_pd_df(items: List[Dict], how: str = 'all') -> pd.DataFrame:
+    """
+    Converts a list of dictionaries to a pandas DataFrame, dropping NA values.
+
+    Args:
+        items (List[Dict]): A list of dictionaries to be converted to a DataFrame.
+        how (str): How to handle NA values. Options are 'all' to drop rows with all NA values,
+                   'any' to drop rows with any NA values, or 'none' to keep all rows.
+
+    Returns:
+        pd.DataFrame: A pandas DataFrame containing the data from the input list of dictionaries.
+    """
+    df = pd.DataFrame(items).dropna(how=how)
+    df.reset_index(drop=True, inplace=True)
+    return df
+
+def pd_row_to_node(row: pd.Series):
+    """
+    Converts a pandas Series row to a DataNode object with structured data.
+
+    Args:
+        row (pd.Series): A pandas Series containing row data to be converted.
+
+    Returns:
+        DataNode: A DataNode object containing structured data from the input pandas Series.
+    """
+    dict_ = row.to_dict()
+    dict_['datetime'] = str(timestamp_to_datetime(dict_['datetime']))
+    dict_['content'] = {'headline': dict_.pop('headline'), 'summary': dict_.pop('summary')}
+    dict_['metadata'] = {'datetime': dict_.pop('datetime'), 'url': dict_.pop('url'), 'id': dict_.pop('id')}
+    return DataNode.from_dict(dict_)
+
+def expand_df_datetime(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Expands a pandas DataFrame with a datetime column into separate year, month, and day columns.
+
+    Args:
+        df (pd.DataFrame): The pandas DataFrame containing a 'datetime' column to be expanded.
+
+    Returns:
+        pd.DataFrame: A pandas DataFrame with 'year', 'month', and 'day' columns.
+    """
+    df_expanded = df.copy()
+    if df_expanded['datetime'].dtype == int:
+        df_expanded['datetime'] = df_expanded['datetime'].apply(lambda x: timestamp_to_datetime(x))
+
+    df_expanded.insert(0, 'year', df_expanded['datetime'].dt.year)
+    df_expanded.insert(1, 'month', df_expanded['datetime'].dt.month)
+    df_expanded.insert(2, 'day', df_expanded['datetime'].dt.day)
+    df_expanded.drop('datetime', axis=1, inplace=True)
+
+    return df_expanded