lionagi 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

lionagi/integrations/pandas_/remove_rows.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+import pandas as pd
+
+from .to_df import to_df
+
+
+def remove_rows(
+    df: pd.DataFrame,
+    rows: int | slice | list[int],
+    from_end: bool = False,
+    reset_index: bool = False,
+) -> pd.DataFrame:
+    """
+    Removes specified rows from a DataFrame.
+
+    Args:
+        df: The DataFrame from which to remove rows.
+        rows: The row(s) to remove. Can be an integer, slice, or list of integers.
+        from_end: If True, count rows from the end of the DataFrame.
+        reset_index: If True, reset the index after removing rows.
+
+    Returns:
+        A DataFrame with the specified rows removed.
+
+    Raises:
+        ValueError: If the specified rows are invalid.
+
+    Example:
+        >>> df = pd.DataFrame({'A': range(10)})
+        >>> remove_rows(df, [0, 2, 4], reset_index=True)
+           A
+        0  1
+        1  3
+        2  5
+        3  6
+        4  7
+        5  8
+        6  9
+    """
+    if df.empty:
+        return df.copy()
+
+    if isinstance(rows, int):
+        rows = [rows]
+    elif isinstance(rows, slice):
+        rows = list(range(*rows.indices(len(df))))
+
+    if from_end:
+        rows = [len(df) - 1 - r for r in rows]
+
+    try:
+        result = df.drop(df.index[rows])
+    except IndexError as e:
+        raise ValueError(f"Invalid row selection: {e}") from e
+
+    if reset_index:
+        result = result.reset_index(drop=True)
+
+    return to_df(result)

lionagi/integrations/pandas_/replace_keywords.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+import pandas as pd
+
+from .to_df import to_df
+
+
+def replace_keywords(
+    df: pd.DataFrame,
+    keyword: str | dict[str, str],
+    replacement: str | None = None,
+    *,
+    column: str | list[str] = "content",
+    case_sensitive: bool = False,
+    regex: bool = False,
+    inplace: bool = False,
+) -> pd.DataFrame | None:
+    """
+    Replaces occurrences of specified keyword(s) with replacement string(s) in DataFrame column(s).
+
+    Args:
+        df: The DataFrame to modify.
+        keyword: The keyword to be replaced. Can be a string or a dictionary of {old: new} pairs.
+        replacement: The string to replace the keyword with. Ignored if keyword is a dict.
+        column: The column(s) in which to perform the replacement. Can be a string or list of strings.
+        case_sensitive: If True, performs a case-sensitive replacement.
+        regex: If True, treat keyword(s) as regular expressions.
+        inplace: If True, modifies the DataFrame in place and returns None.
+
+    Returns:
+        Modified DataFrame if inplace is False, None otherwise.
+
+    Example:
+        >>> df = pd.DataFrame({'content': ['apple pie', 'banana split', 'cherry pie']})
+        >>> replace_keyword(df, {'pie': 'tart', 'split': 'smoothie'}, column='content')
+           content
+        0  apple tart
+        1  banana smoothie
+        2  cherry tart
+    """
+    df_ = df if inplace else df.copy()
+
+    if isinstance(column, str):
+        column = [column]
+
+    if isinstance(keyword, dict):
+        for col in column:
+            df_[col] = df_[col].replace(
+                to_replace=keyword,
+                regex=regex,
+                case=case_sensitive,
+            )
+    else:
+        for col in column:
+            df_[col] = df_[col].replace(
+                to_replace=keyword,
+                value=replacement,
+                regex=regex,
+                case=case_sensitive,
+            )
+
+    if inplace:
+        return None
+    return to_df(df_)

lionagi/integrations/pandas_/save.py

@@ -0,0 +1,131 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+from pathlib import Path
+from typing import Any
+
+from lionagi.integrations.pandas_.to_df import to_df
+from lionagi.libs.file.types import create_path
+
+
+def to_csv(
+    input_: Any,
+    /,
+    *,
+    directory: str | Path,
+    filename: str,
+    timestamp: bool = False,
+    random_hash_digits: int = 0,
+    drop_how: str = "all",
+    drop_kwargs: dict[str, Any] | None = None,
+    reset_index: bool = True,
+    concat_kwargs: dict[str, Any] | None = None,
+    df_kwargs: dict[str, Any] | None = None,
+    path_kwargs: dict[str, Any] | None = None,
+    verbose: bool = False,
+) -> None:
+    """
+    Convert input to a DataFrame and save it as a CSV file.
+
+    Args:
+        input_: The input data to convert to a DataFrame.
+        directory: The directory to save the CSV file.
+        filename: The name of the CSV file (without extension).
+        timestamp: If True, add a timestamp to the filename.
+        random_hash_digits: Number of random hash digits to add to the filename.
+        drop_how: How to drop NA values in the DataFrame.
+        drop_kwargs: Additional keyword arguments for dropna().
+        reset_index: If True, reset the index of the DataFrame.
+        concat_kwargs: Keyword arguments for pandas.concat().
+        df_kwargs: Additional keyword arguments for to_df().
+        path_kwargs: Additional keyword arguments for create_path().
+        verbose: If True, print the file path after saving.
+
+    Returns:
+        None
+    """
+    df = to_df(
+        input_,
+        drop_how=drop_how,
+        drop_kwargs=drop_kwargs,
+        reset_index=reset_index,
+        concat_kwargs=concat_kwargs,
+        **(df_kwargs or {}),
+    )
+
+    fp = create_path(
+        directory=directory,
+        filename=filename,
+        timestamp=timestamp,
+        random_hash_digits=random_hash_digits,
+        extension="csv",
+        **(path_kwargs or {}),
+    )
+
+    df.to_csv(fp, index=False)
+
+    if verbose:
+        print(f"Data saved to {fp}")
+
+
+def to_excel(
+    input_: Any,
+    /,
+    *,
+    directory: str | Path,
+    filename: str,
+    timestamp: bool = False,
+    random_hash_digits: int = 0,
+    drop_how: str = "all",
+    drop_kwargs: dict[str, Any] | None = None,
+    reset_index: bool = True,
+    concat_kwargs: dict[str, Any] | None = None,
+    df_kwargs: dict[str, Any] | None = None,
+    path_kwargs: dict[str, Any] | None = None,
+    verbose: bool = False,
+) -> None:
+    """
+    Convert input to a DataFrame and save it as an Excel file.
+
+    Args:
+        input_: The input data to convert to a DataFrame.
+        directory: The directory to save the Excel file.
+        filename: The name of the Excel file (without extension).
+        timestamp: If True, add a timestamp to the filename.
+        random_hash_digits: Number of random hash digits to add to the filename.
+        drop_how: How to drop NA values in the DataFrame.
+        drop_kwargs: Additional keyword arguments for dropna().
+        reset_index: If True, reset the index of the DataFrame.
+        concat_kwargs: Keyword arguments for pandas.concat().
+        df_kwargs: Additional keyword arguments for to_df().
+        path_kwargs: Additional keyword arguments for create_path().
+        verbose: If True, print the file path after saving.
+
+    Returns:
+        None
+    """
+    df = to_df(
+        input_,
+        drop_how=drop_how,
+        drop_kwargs=drop_kwargs,
+        reset_index=reset_index,
+        concat_kwargs=concat_kwargs,
+        **(df_kwargs or {}),
+    )
+
+    fp = create_path(
+        directory=directory,
+        filename=filename,
+        timestamp=timestamp,
+        random_hash_digits=random_hash_digits,
+        extension="xlsx",
+        **(path_kwargs or {}),
+    )
+
+    df.to_excel(fp, index=False)
+
+    if verbose:
+        print(f"Data saved to {fp}")
+
+
+__all__ = ["to_csv", "to_excel"]

lionagi/integrations/pandas_/search_keywords.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+import pandas as pd
+
+from .to_df import to_df
+
+
+def search_dataframe_keywords(
+    df: pd.DataFrame,
+    keywords: str | list[str],
+    *,
+    column: str | list[str] = "content",
+    case_sensitive: bool = False,
+    reset_index: bool = False,
+    dropna: bool = False,
+    regex: bool = False,
+    match_all: bool = False,
+) -> pd.DataFrame:
+    """
+    Filters a DataFrame for rows where specified column(s) contain given keywords.
+
+    Args:
+        df: The DataFrame to search through.
+        keywords: A keyword or list of keywords to search for.
+        column: The column(s) to perform the search in. Can be a string or list of strings.
+        case_sensitive: Whether the search should be case-sensitive.
+        reset_index: Whether to reset the DataFrame's index after filtering.
+        dropna: Whether to drop rows with NA values before searching.
+        regex: If True, treat keywords as regular expressions.
+        match_all: If True, all keywords must be present for a match.
+
+    Returns:
+        A filtered DataFrame containing only rows where the specified column(s) contain
+        any (or all) of the provided keywords.
+
+    Example:
+        >>> df = pd.DataFrame({'content': ['apple pie', 'banana split', 'cherry pie']})
+        >>> search_keywords(df, ['pie', 'cherry'], match_all=True)
+           content
+        2  cherry pie
+    """
+    if dropna:
+        df = df.dropna(subset=[column] if isinstance(column, str) else column)
+
+    if isinstance(keywords, str):
+        keywords = [keywords]
+
+    if isinstance(column, str):
+        column = [column]
+
+    mask = pd.DataFrame(index=df.index)
+    for col in column:
+        col_mask = pd.Series(False, index=df.index)
+        for keyword in keywords:
+            keyword_mask = df[col].str.contains(
+                keyword, case=case_sensitive, regex=regex, na=False
+            )
+            col_mask = col_mask | keyword_mask
+        mask[col] = col_mask
+
+    final_mask = mask.all(axis=1) if match_all else mask.any(axis=1)
+
+    result = df[final_mask]
+
+    if reset_index:
+        result = result.reset_index(drop=True)
+
+    return to_df(result)

lionagi/integrations/pandas_/to_df.py

@@ -0,0 +1,196 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+from typing import Any, Literal
+
+from pandas import DataFrame, Series, concat
+from pandas.core.generic import NDFrame
+
+from lionagi.libs.parse import to_dict, to_list
+
+
+def to_df(
+    input_: Any,
+    /,
+    *,
+    drop_how: Literal["any", "all"] = "all",
+    drop_kwargs: dict[str, Any] | None = None,
+    reset_index: bool = True,
+    concat_kwargs: dict[str, Any] | None = None,
+    **kwargs: Any,
+) -> DataFrame:
+    """
+    Convert various input types to a pandas DataFrame.
+
+    This function attempts to convert the input to a DataFrame, handling
+    both single objects and lists of objects. It can process various input
+    types including dictionaries, lists, and existing pandas objects.
+
+    Args:
+        input_: The input data to convert to a DataFrame.
+        drop_how: How to drop NA values. Either "any" or "all".
+        drop_kwargs: Additional keyword arguments for dropna().
+        reset_index: Whether to reset the index of the resulting DataFrame.
+        concat_kwargs: Keyword arguments for pandas.concat() when dealing with lists.
+        **kwargs: Additional keyword arguments passed to DataFrame constructor.
+
+    Returns:
+        A pandas DataFrame created from the input data.
+
+    Raises:
+        ValueError: If the input cannot be converted to a DataFrame.
+
+    Example:
+        >>> data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
+        >>> df = to_df(data)
+        >>> print(df)
+           a  b
+        0  1  2
+        1  3  4
+    """
+    if not isinstance(input_, list):
+        try:
+            return general_to_df(
+                input_,
+                drop_how=drop_how,
+                drop_kwargs=drop_kwargs,
+                reset_index=reset_index,
+                **kwargs,
+            )
+        except ValueError:
+            input_ = [input_]
+
+    if isinstance(input_, list):
+        try:
+            return _list_to_df(
+                input_,
+                drop_how=drop_how,
+                drop_kwargs=drop_kwargs,
+                reset_index=reset_index,
+                concat_kwargs=concat_kwargs or {},
+                **kwargs,
+            )
+        except ValueError:
+            try:
+                _d = [to_dict(i) for i in input_]
+                return _list_to_df(
+                    _d,
+                    drop_how=drop_how,
+                    drop_kwargs=drop_kwargs,
+                    reset_index=reset_index,
+                    concat_kwargs=concat_kwargs or {},
+                    **kwargs,
+                )
+            except ValueError:
+                raise ValueError(
+                    "Error converting input_ to DataFrame"
+                ) from None
+
+
+def general_to_df(
+    input_: Any,
+    /,
+    *,
+    drop_how: str = "all",
+    drop_kwargs: dict[str, Any] | None = None,
+    reset_index: bool = True,
+    **kwargs: Any,
+) -> DataFrame:
+
+    try:
+        return _general_to_df(
+            input_,
+            drop_how=drop_how,
+            drop_kwargs=drop_kwargs,
+            reset_index=reset_index,
+            **kwargs,
+        )
+    except ValueError:
+        try:
+            _d = to_dict(input_)
+            return _general_to_df(
+                _d,
+                drop_how=drop_how,
+                drop_kwargs=drop_kwargs,
+                reset_index=reset_index,
+                **kwargs,
+            )
+        except ValueError:
+            raise ValueError("Error converting input_ to DataFrame") from None
+
+
+def _general_to_df(
+    input_: Any,
+    /,
+    *,
+    drop_how: str = "all",
+    drop_kwargs: dict[str, Any] | None = None,
+    reset_index: bool = True,
+    **kwargs: Any,
+) -> DataFrame:
+    if drop_kwargs is None:
+        drop_kwargs = {}
+
+    try:
+        df: DataFrame = DataFrame(input_, **kwargs)
+
+        if "thresh" not in drop_kwargs:
+            drop_kwargs["how"] = drop_how
+        df = df.dropna(**drop_kwargs)
+        return df.reset_index(drop=True) if reset_index else df
+    except Exception as e:
+        raise ValueError(f"Error converting input_ to DataFrame: {e}") from e
+
+
+def _list_to_df(
+    input_: list,
+    /,
+    *,
+    drop_how: str = "all",
+    drop_kwargs: dict | None = None,
+    reset_index: bool = True,
+    concat_kwargs: dict | None = None,
+    **kwargs,
+) -> DataFrame:
+    if not input_:
+        return DataFrame()
+
+    if not isinstance(input_[0], (DataFrame, Series, NDFrame)):
+        if drop_kwargs is None:
+            drop_kwargs = {}
+        try:
+            df: DataFrame = DataFrame(input_, **kwargs)
+            if "thresh" not in drop_kwargs:
+                drop_kwargs["how"] = drop_how
+            df = df.dropna(**drop_kwargs)
+            return df.reset_index(drop=True) if reset_index else df
+        except Exception as e:
+            raise ValueError(
+                f"Error converting input_ to DataFrame: {e}"
+            ) from e
+
+    if drop_kwargs is None:
+        drop_kwargs = {}
+    try:
+        config = concat_kwargs.copy()
+        config.pop("axis", None)
+        df = concat(
+            input_,
+            axis=1 if all(isinstance(i, Series) for i in input_) else 0,
+            **concat_kwargs,
+        )
+    except Exception as e1:
+        try:
+            input_ = to_list(input_, dropna=True, flatten=True)
+            df = input_[0]
+            if len(input_) > 1:
+                for i in input_[1:]:
+                    df = concat([df, i], **concat_kwargs)
+        except Exception as e2:
+            raise ValueError(
+                f"Error converting input_ to DataFrame: {e1}, {e2}"
+            ) from e2
+
+    drop_kwargs["how"] = drop_how
+    df.dropna(**drop_kwargs, inplace=True)
+    return df.reset_index(drop=True) if reset_index else df

lionagi/integrations/pandas_/update_cells.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+from typing import Any
+
+import pandas as pd
+
+from .to_df import to_df
+
+
+def update_cells(
+    df: pd.DataFrame,
+    updates: dict[tuple[int | str, int | str], Any],
+    create_missing: bool = False,
+) -> pd.DataFrame:
+    """
+    Updates multiple cells in a DataFrame based on a dictionary of updates.
+
+    Args:
+        df: The DataFrame to update.
+        updates: A dictionary where keys are (row, column) tuples and values are the new values.
+                 Rows and columns can be specified by integer position or label.
+        create_missing: If True, create new columns if they don't exist.
+
+    Returns:
+        The updated DataFrame.
+
+    Raises:
+        KeyError: If a specified row or column doesn't exist and create_missing is False.
+
+    Example:
+        >>> df = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
+        >>> updates = {(0, 'A'): 10, (1, 'B'): 40, (0, 'C'): 100}
+        >>> update_cells(df, updates, create_missing=True)
+            A   B    C
+        0  10   3  100
+        1   2  40  NaN
+    """
+    df_copy = df.copy()
+
+    for (row, col), value in updates.items():
+        try:
+            if isinstance(col, str) and col not in df_copy.columns:
+                if create_missing:
+                    df_copy[col] = None
+                else:
+                    raise KeyError(f"Column '{col}' does not exist.")
+
+            df_copy.loc[row, col] = value
+        except KeyError as e:
+            if not create_missing:
+                raise KeyError(f"Invalid row or column: {e}") from e
+
+    return to_df(df_copy)