scitex 2.0.0__py2.py3-none-any.whl

scitex/pd/_merge_columns.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-05 07:37:09 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/pd/_merge_columns.py
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-10-07 12:03:29 (ywatanabe)"
+# ./src/scitex/pd/_merge_cols.py
+
+from typing import Union, List, Tuple
+import pandas as pd
+
+
+def merge_columns(
+    df: pd.DataFrame,
+    *args: Union[str, List[str], Tuple[str, ...]],
+    sep: str = None,
+    sep1: str = "_",
+    sep2: str = "-",
+    name: str = "merged"
+) -> pd.DataFrame:
+    """Creates a new column by joining specified columns.
+
+    Example
+    -------
+    >>> df = pd.DataFrame({
+    ...     'A': [0, 5, 10],
+    ...     'B': [1, 6, 11],
+    ...     'C': [2, 7, 12]
+    ... })
+    >>> # Simple concatenation with separator
+    >>> merge_columns(df, 'A', 'B', sep=' ')
+       A  B  C    A_B
+    0  0  1  2    0 1
+    1  5  6  7    5 6
+    2 10 11 12  10 11
+
+    >>> # With column labels
+    >>> merge_columns(df, 'A', 'B', sep1='_', sep2='-')
+       A  B  C        A_B
+    0  0  1  2    A-0_B-1
+    1  5  6  7    A-5_B-6
+    2 10 11 12  A-10_B-11
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Input DataFrame
+    *args : Union[str, List[str], Tuple[str, ...]]
+        Column names to join
+    sep : str, optional
+        Simple separator for values only (overrides sep1/sep2)
+    sep1 : str, optional
+        Separator between column-value pairs, by default "_"
+    sep2 : str, optional
+        Separator between column name and value, by default "-"
+    name : str, optional
+        Name for the merged column, by default "merged"
+
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with added merged column
+    """
+    _df = df.copy()
+    columns = args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args
+
+    if not columns:
+        raise ValueError("No columns specified for merging")
+
+    if not all(col in _df.columns for col in columns):
+        missing = [col for col in columns if col not in _df.columns]
+        raise KeyError(f"Columns not found in DataFrame: {missing}")
+
+    # Handle empty DataFrame case
+    if len(_df) == 0:
+        # Determine column name
+        if name == "merged" and sep is not None:
+            new_col_name = "_".join(columns)
+        else:
+            new_col_name = name
+        # Create empty Series with the correct name
+        _df[new_col_name] = pd.Series(dtype=str)
+        return _df
+
+    if sep is not None:
+        # Simple value concatenation
+        merged_col = (
+            _df[list(columns)]
+            .astype(str)
+            .apply(
+                lambda row: sep.join(row.values),
+                axis=1,
+            )
+        )
+    else:
+        # Concatenation with column labels
+        merged_col = _df[list(columns)].apply(
+            lambda row: sep1.join(f"{col}{sep2}{val}" for col, val in row.items()),
+            axis=1,
+        )
+
+    # Determine column name
+    if name == "merged" and sep is not None:
+        # When using simple separator and default name, use joined column names
+        new_col_name = "_".join(columns)
+    else:
+        # Use provided name or default
+        new_col_name = name
+
+    _df[new_col_name] = merged_col
+    return _df
+
+
+merge_cols = merge_columns
+
+# EOF
+
+# #!./env/bin/python3
+# # -*- coding: utf-8 -*-
+# # Time-stamp: "2024-10-07 12:03:29 (ywatanabe)"
+# # ./src/scitex/pd/_merge_cols.py
+
+
+# def merge_columns(df, *args, sep1="_", sep2="-", name="merged"):
+#     """
+#     Join specified columns with their labels.
+
+#     Example:
+#         import pandas as pd
+#         import numpy as np
+
+#         df = pd.DataFrame(
+#             data=np.arange(25).reshape(5, 5),
+#             columns=["A", "B", "C", "D", "E"],
+#         )
+
+#         df1 = merge_columns(df, "A", "B", sep1="_", sep2="-")
+#         df2 = merge_columns(df, ["A", "B"], sep1="_", sep2="-")
+#         assert (df1 == df2).all().all() # True
+
+#         #     A   B   C   D   E        A_B
+#         # 0   0   1   2   3   4    A-0_B-1
+#         # 1   5   6   7   8   9    A-5_B-6
+#         # 2  10  11  12  13  14  A-10_B-11
+#         # 3  15  16  17  18  19  A-15_B-16
+#         # 4  20  21  22  23  24  A-20_B-21
+
+
+#     Parameters
+#     ----------
+#     df : pandas.DataFrame
+#         Input DataFrame
+#     *args : str or list
+#         Column names to join, either as separate arguments or a single list
+#     sep1 : str, optional
+#         Separator for joining column names, default "_"
+#     sep2 : str, optional
+#         Separator between column name and value, default "-"
+
+#     Returns
+#     -------
+#     pandas.DataFrame
+#         DataFrame with added merged column
+#     """
+#     _df = df.copy()
+#     columns = (
+#         args[0]
+#         if len(args) == 1 and isinstance(args[0], (list, tuple))
+#         else args
+#     )
+#     merged_col = _df[list(columns)].apply(
+#         lambda row: sep1.join(f"{col}{sep2}{val}" for col, val in row.items()),
+#         axis=1,
+#     )
+
+#     new_col_name = sep1.join(columns) if not name else str(name)
+#     _df[new_col_name] = merged_col
+#     return _df
+
+
+# merge_cols = merge_columns
+
+# # def merge_columns(_df, *columns):
+# #     """
+# #     Add merged columns in string.
+
+# #     DF = pd.DataFrame(data=np.arange(25).reshape(5,5),
+# #                       columns=["A", "B", "C", "D", "E"],
+# #     )
+
+# #     print(DF)
+
+# #     # A   B   C   D   E
+# #     # 0   0   1   2   3   4
+# #     # 1   5   6   7   8   9
+# #     # 2  10  11  12  13  14
+# #     # 3  15  16  17  18  19
+# #     # 4  20  21  22  23  24
+
+# #     print(merge_columns(DF, "A", "B", "C"))
+
+# #     #     A   B   C   D   E     A_B_C
+# #     # 0   0   1   2   3   4     0_1_2
+# #     # 1   5   6   7   8   9     5_6_7
+# #     # 2  10  11  12  13  14  10_11_12
+# #     # 3  15  16  17  18  19  15_16_17
+# #     # 4  20  21  22  23  24  20_21_22
+# #     """
+# #     from copy import deepcopy
+
+# #     df = deepcopy(_df)
+# #     merged = deepcopy(df[columns[0]])  # initialization
+# #     for c in columns[1:]:
+# #         merged = scitex.ml.utils.merge_labels(list(merged), deepcopy(df[c]))
+# #     df.loc[:, scitex.gen.connect_strs(columns)] = merged
+# #     return df
+
+
+# EOF

scitex/pd/_mv.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-05 07:39:12 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/pd/_mv.py
+
+
+def mv(df, key, position, axis=1):
+    """
+    Move a row or column to a specified position in a DataFrame.
+
+    Args:
+    df (pandas.DataFrame): The input DataFrame.
+    key (str): The label of the row or column to move.
+    position (int): The position to move the row or column to.
+    axis (int, optional): 0 for rows, 1 for columns. Defaults to 1.
+
+    Returns:
+    pandas.DataFrame: A new DataFrame with the row or column moved.
+    """
+    if axis == 0:
+        items = df.index.tolist()
+    else:
+        items = df.columns.tolist()
+    items.remove(key)
+
+    if position < 0:
+        position += len(items) + 1
+
+    items.insert(position, key)
+    return df.reindex(items, axis=axis)
+
+
+def mv_to_first(df, key, axis=1):
+    """
+    Move a row or column to the first position in a DataFrame.
+
+    Args:
+    df (pandas.DataFrame): The input DataFrame.
+    key (str): The label of the row or column to move.
+    axis (int, optional): 0 for rows, 1 for columns. Defaults to 1.
+
+    Returns:
+    pandas.DataFrame: A new DataFrame with the row or column moved to the first position.
+    """
+    return mv(df, key, 0, axis)
+
+
+def mv_to_last(df, key, axis=1):
+    """
+    Move a row or column to the last position in a DataFrame.
+
+    Args:
+    df (pandas.DataFrame): The input DataFrame.
+    key (str): The label of the row or column to move.
+    axis (int, optional): 0 for rows, 1 for columns. Defaults to 1.
+
+    Returns:
+    pandas.DataFrame: A new DataFrame with the row or column moved to the last position.
+    """
+    return mv(df, key, -1, axis)
+
+
+# EOF

scitex/pd/_replace.py

@@ -0,0 +1,62 @@
+#!./env/bin/python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-08-29 23:08:35 (ywatanabe)"
+# ./src/scitex/pd/_replace.py
+
+
+def replace(dataframe, old_value, new_value=None, regex=False, cols=None):
+    """
+    Replace values in a DataFrame.
+
+    Example
+    -------
+    import pandas as pd
+    df = pd.DataFrame({'A': ['abc-123', 'def-456'], 'B': ['ghi-789', 'jkl-012']})
+
+    # Replace single value
+    df_replaced = replace(df, 'abc', 'xyz')
+
+    # Replace with dictionary
+    replace_dict = {'-': '_', '1': 'one'}
+    df_replaced = replace(df, replace_dict, cols=['A'])
+    print(df_replaced)
+
+    Parameters
+    ----------
+    dataframe : pandas.DataFrame
+        Input DataFrame to modify.
+    old_value : str, dict
+        If str, the value to replace (requires new_value).
+        If dict, mapping of old values (keys) to new values (values).
+    new_value : str, optional
+        New value to replace old_value with. Required if old_value is str.
+    regex : bool, optional
+        If True, treat replacement keys as regular expressions. Default is False.
+    cols : list of str, optional
+        List of column names to apply replacements. If None, apply to all columns.
+
+    Returns
+    -------
+    pandas.DataFrame
+        DataFrame with specified replacements applied.
+    """
+    dataframe = dataframe.copy()
+
+    # Handle different input formats
+    if isinstance(old_value, dict):
+        replace_dict = old_value
+    else:
+        if new_value is None:
+            raise ValueError("new_value must be provided when old_value is not a dict")
+        replace_dict = {old_value: new_value}
+
+    # Apply replacements to all columns if cols not specified
+    if cols is None:
+        # Use pandas replace method for all columns
+        return dataframe.replace(replace_dict, regex=regex)
+    else:
+        # Apply to specific columns
+        for column in cols:
+            if column in dataframe.columns:
+                dataframe[column] = dataframe[column].replace(replace_dict, regex=regex)
+        return dataframe

scitex/pd/_round.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-10-06 11:13:00 (ywatanabe)"
+# /home/ywatanabe/proj/_scitex_repo_openhands/src/scitex/pd/_round.py
+
+import numpy as np
+import pandas as pd
+
+
+def round(df: pd.DataFrame, factor: int = 3) -> pd.DataFrame:
+    """
+    Round numeric values in a DataFrame to a specified number of decimal places.
+
+    Example
+    -------
+    >>> df = pd.DataFrame({'A': [1.23456, 2.34567], 'B': ['abc', 'def'], 'C': [3, 4]})
+    >>> round(df, 2)
+          A    B  C
+    0  1.23  abc  3
+    1  2.35  def  4
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Input DataFrame
+    factor : int, optional
+        Number of decimal places to round to (default is 3)
+
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with rounded numeric values
+    """
+
+    def custom_round(column):
+        # Skip non-numeric types like datetime, categorical, string
+        if pd.api.types.is_datetime64_any_dtype(column):
+            return column
+        if pd.api.types.is_categorical_dtype(column):
+            return column
+        if pd.api.types.is_string_dtype(column):
+            return column
+        # Note: boolean types are allowed to be converted to numeric
+        if pd.api.types.is_object_dtype(column) and not pd.api.types.is_numeric_dtype(column) and not pd.api.types.is_bool_dtype(column):
+            return column
+            
+        try:
+            # Handle boolean columns explicitly
+            if pd.api.types.is_bool_dtype(column):
+                return column.astype(int)
+                
+            numeric_column = pd.to_numeric(column, errors="coerce")
+            if np.issubdtype(numeric_column.dtype, np.integer):
+                return numeric_column.astype(int)
+            
+            # For float columns, round first
+            rounded = numeric_column.round(factor)
+            
+            # If factor is 0 and all values are whole numbers, convert to int
+            if factor == 0 and (rounded % 1 == 0).all() and not rounded.isna().any():
+                return rounded.astype(int)
+                
+            return rounded
+            
+        except (ValueError, TypeError):
+            return column
+
+    return df.apply(custom_round)
+
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-10-05 20:40:32 (ywatanabe)"
+# /home/ywatanabe/proj/_scitex_repo_openhands/src/scitex/pd/_round.py
+
+# import numpy as np
+
+# def round(df, factor=3):
+#     return df.apply(lambda x: x.round(factor) if np.issubdtype(x.dtype, np.number) else x)
+
+
+# def round(df, factor=3):
+#     def custom_round(x):
+#         try:
+#             numeric_x = pd.to_numeric(x, errors='raise')
+#             if np.issubdtype(numeric_x.dtype, np.integer):
+#                 return numeric_x
+#             else:
+#                 return numeric_x.apply(lambda y: float(f'{y:.{factor}g}'))
+#         except (ValueError, TypeError):
+#             return x
+
+#     return df.apply(custom_round)

scitex/pd/_slice.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-05 07:45:00 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/pd/_slice.py
+
+from typing import Dict, Union, List, Optional
+import builtins
+
+import pandas as pd
+
+from ._find_indi import find_indi
+
+
+def slice(
+    df: pd.DataFrame,
+    conditions: Union[
+        builtins.slice, Dict[str, Union[str, int, float, List]], None
+    ] = None,
+    columns: Optional[List[str]] = None,
+) -> pd.DataFrame:
+    """Slices DataFrame rows and/or columns.
+
+    Example
+    -------
+    >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': ['x', 'y', 'x']})
+    >>> # Slice by row indices
+    >>> result = slice(df, slice(0, 2))
+    >>> # Slice by conditions
+    >>> result = slice(df, {'A': [1, 2], 'B': 'x'})
+    >>> # Slice columns
+    >>> result = slice(df, columns=['A'])
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Input DataFrame to slice
+    conditions : slice, Dict, or None
+        Either a slice object for row indices, or a dictionary of column conditions
+    columns : List[str], optional
+        List of column names to select
+
+    Returns
+    -------
+    pd.DataFrame
+        Sliced DataFrame
+    """
+    result = df.copy()
+
+    # Handle row slicing
+    if isinstance(conditions, builtins.slice):
+        result = result.iloc[conditions]
+    elif isinstance(conditions, dict):
+        indices = find_indi(result, conditions)
+        result = result.loc[indices]
+
+    # Handle column slicing
+    if columns is not None:
+        result = result[columns]
+
+    return result
+
+
+# EOF

scitex/pd/_sort.py

@@ -0,0 +1,91 @@
+#!./env/bin/python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-08-25 09:35:39 (ywatanabe)"
+# ./src/scitex/pd/_sort.py
+
+import pandas as pd
+
+
+def sort(
+    dataframe,
+    by=None,
+    ascending=True,
+    inplace=False,
+    kind="quicksort",
+    na_position="last",
+    ignore_index=False,
+    key=None,
+    orders=None,
+):
+    """
+    Sort DataFrame by specified column(s) with optional custom ordering and column reordering.
+
+    Example
+    -------
+    import pandas as pd
+    df = pd.DataFrame({'A': ['foo', 'bar', 'baz'], 'B': [3, 2, 1]})
+    custom_order = {'A': ['bar', 'baz', 'foo']}
+    sorted_df = sort(df, by=None, orders=custom_order)
+    print(sorted_df)
+
+    Parameters
+    ----------
+    dataframe : pandas.DataFrame
+        The DataFrame to sort.
+    by : str or list of str, optional
+        Name(s) of column(s) to sort by.
+    ascending : bool or list of bool, default True
+        Sort ascending vs. descending.
+    inplace : bool, default False
+        If True, perform operation in-place.
+    kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, default 'quicksort'
+        Choice of sorting algorithm.
+    na_position : {'first', 'last'}, default 'last'
+        Puts NaNs at the beginning if 'first'; 'last' puts NaNs at the end.
+    ignore_index : bool, default False
+        If True, the resulting axis will be labeled 0, 1, …, n - 1.
+    key : callable, optional
+        Apply the key function to the values before sorting.
+    orders : dict, optional
+        Dictionary of column names and their custom sort orders.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Sorted DataFrame with reordered columns.
+    """
+    if orders:
+        by = [by] if isinstance(by, str) else list(orders.keys()) if by is None else by
+
+        def apply_custom_order(column):
+            return (
+                pd.Categorical(column, categories=orders[column.name], ordered=True)
+                if column.name in orders
+                else column
+            )
+
+        key = apply_custom_order
+    elif isinstance(by, str):
+        by = [by]
+
+    sorted_df = dataframe.sort_values(
+        by=by,
+        ascending=ascending,
+        inplace=False,
+        kind=kind,
+        na_position=na_position,
+        ignore_index=ignore_index,
+        key=key,
+    )
+
+    # Reorder columns
+    if by:
+        other_columns = [col for col in sorted_df.columns if col not in by]
+        sorted_df = sorted_df[by + other_columns]
+
+    if inplace:
+        dataframe.update(sorted_df)
+        dataframe.reindex(columns=sorted_df.columns)
+        return dataframe
+    else:
+        return sorted_df

scitex/pd/_to_numeric.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-08 04:35:31 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/pd/_to_numeric.py
+
+import pandas as pd
+
+
+def to_numeric(df, errors="coerce"):
+    """Convert all possible columns in a DataFrame to numeric types.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Input DataFrame
+    errors : str, optional
+        How to handle errors. 'coerce' (default) converts invalid values to NaN,
+        'ignore' leaves non-numeric columns unchanged, 'raise' raises exceptions.
+
+    Returns
+    -------
+    pd.DataFrame
+        DataFrame with numeric columns converted
+    """
+    df_copy = df.copy()
+    for col in df_copy.columns:
+        # First try to convert
+        original_col = df_copy[col]
+        converted_col = pd.to_numeric(df_copy[col], errors="coerce")
+
+        # Check if conversion resulted in all NaN when original had values
+        if converted_col.isna().all() and not original_col.isna().all():
+            # This is likely a pure string column
+            if errors == "ignore":
+                # Keep original for pure string columns
+                continue
+            else:
+                # For coerce, still apply it
+                df_copy[col] = converted_col
+        elif not converted_col.equals(original_col):
+            # Conversion changed something
+            if errors == "ignore":
+                # Only convert if it doesn't introduce new NaNs
+                if converted_col.isna().sum() == original_col.isna().sum():
+                    df_copy[col] = converted_col
+            elif errors == "coerce":
+                df_copy[col] = converted_col
+            elif errors == "raise":
+                df_copy[col] = pd.to_numeric(df_copy[col], errors="raise")
+    return df_copy
+
+
+# EOF