scitex 2.0.0__py2.py3-none-any.whl

scitex/pd/_find_indi.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-05 08:11:05 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/pd/_find_indi.py
+
+from typing import Dict, List, Union
+
+import pandas as pd
+
+
+# def find_indi(df: pd.DataFrame, conditions: Dict[str, Union[str, int, float, List]]) -> pd.Series:
+#     """Finds indices of rows that satisfy all given conditions in a DataFrame.
+
+#     Example
+#     -------
+#     >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': ['x', 'y', 'x']})
+#     >>> conditions = {'A': [1, 2], 'B': 'x'}
+#     >>> result = find_indi(df, conditions)
+#     >>> print(result)
+#     0     True
+#     1    False
+#     2    False
+#     dtype: bool
+
+#     Parameters
+#     ----------
+#     df : pd.DataFrame
+#         Input DataFrame to search in
+#     conditions : Dict[str, Union[str, int, float, List]]
+#         Dictionary of column names and their target values
+
+#     Returns
+#     -------
+#     pd.Series
+#         Boolean Series indicating which rows satisfy all conditions
+
+#     Raises
+#     ------
+#     KeyError
+#         If any column in conditions is not found in DataFrame
+#     """
+#     if not all(col in df.columns for col in conditions):
+#         missing_cols = [col for col in conditions if col not in df.columns]
+#         raise KeyError(f"Columns not found in DataFrame: {missing_cols}")
+
+#     condition_series = []
+#     for key, value in conditions.items():
+#         if isinstance(value, (list, tuple)):
+#             condition_series.append(df[key].isin(value))
+#         else:
+#             condition_series.append(df[key] == value)
+
+#     return pd.concat(condition_series, axis=1).all(axis=1)
+
+
+def find_indi(
+    df: pd.DataFrame, conditions: Dict[str, Union[str, int, float, List]]
+) -> List[int]:
+    """Finds indices of rows that satisfy conditions, handling NaN values.
+
+    Example
+    -------
+    >>> df = pd.DataFrame({'A': [1, 2, None], 'B': ['x', 'y', 'x']})
+    >>> conditions = {'A': [1, None], 'B': 'x'}
+    >>> result = find_indi(df, conditions)
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Input DataFrame
+    conditions : Dict[str, Union[str, int, float, List]]
+        Column conditions
+
+    Returns
+    -------
+    List[int]
+        List of integer indices of matching rows
+    """
+    if not conditions:
+        return []
+
+    if not all(col in df.columns for col in conditions):
+        missing_cols = [col for col in conditions if col not in df.columns]
+        raise KeyError(f"Columns not found in DataFrame: {missing_cols}")
+
+    condition_series = []
+    for key, value in conditions.items():
+        if isinstance(value, (list, tuple)):
+            # Handle NaN in lists
+            has_na = False
+            try:
+                # Check for None
+                if None in value:
+                    has_na = True
+                # Check for pd.NA (may raise TypeError)
+                elif any(v is pd.NA for v in value):
+                    has_na = True
+                # Check for np.nan
+                elif any(pd.isna(v) for v in value):
+                    has_na = True
+            except (TypeError, ValueError):
+                # If any check fails, try alternative approach
+                has_na = any(
+                    pd.isna(v) if not isinstance(v, str) else False for v in value
+                )
+
+            if has_na:
+                condition = df[key].isin(value) | df[key].isna()
+            else:
+                condition = df[key].isin(value)
+        else:
+            # Handle single NaN value
+            if pd.isna(value):
+                condition = df[key].isna()
+            else:
+                condition = df[key] == value
+        condition_series.append(condition)
+
+    if condition_series:
+        mask = pd.concat(condition_series, axis=1).all(axis=1)
+        return df.index[mask].tolist()
+    else:
+        return []
+
+
+# EOF

scitex/pd/_find_pval.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-03 03:25:00 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/pd/_find_pval.py
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-10-06 11:09:07 (ywatanabe)"
+# /home/ywatanabe/proj/_scitex_repo_openhands/src/scitex/stats/_find_pval_col.py
+
+"""
+Functionality:
+    - Identifies column name(s) in a DataFrame or keys in other data structures that correspond to p-values
+Input:
+    - pandas DataFrame, numpy array, list, or dict
+Output:
+    - String or list of strings representing the identified p-value column name(s) or key(s), or None if not found
+Prerequisites:
+    - pandas, numpy libraries
+"""
+
+import re
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+
+def find_pval(
+    data: Union[pd.DataFrame, np.ndarray, List, Dict], multiple: bool = True
+) -> Union[Optional[str], List[str]]:
+    """
+    Find p-value column name(s) or key(s) in various data structures.
+
+    Example:
+    --------
+    >>> df = pd.DataFrame({'p_value': [0.05, 0.01], 'pval': [0.1, 0.001], 'other': [1, 2]})
+    >>> find_pval(df)
+    ['p_value', 'pval']
+    >>> find_pval(df, multiple=False)
+    'p_value'
+
+    Parameters:
+    -----------
+    data : Union[pd.DataFrame, np.ndarray, List, Dict]
+        Data structure to search for p-value column or key
+    multiple : bool, optional
+        If True, return all matches; if False, return only the first match (default is True)
+
+    Returns:
+    --------
+    Union[Optional[str], List[str]]
+        Name(s) of the column(s) or key(s) that match p-value patterns, or None if not found
+    """
+    if isinstance(data, pd.DataFrame):
+        return _find_pval_col(data, multiple)
+    elif isinstance(data, (np.ndarray, list, dict)):
+        return _find_pval(data, multiple)
+    else:
+        raise ValueError("Input must be a pandas DataFrame, numpy array, list, or dict")
+
+
+def _find_pval(
+    data: Union[np.ndarray, List, Dict], multiple: bool
+) -> Union[Optional[str], List[str]]:
+    pattern = re.compile(r"p[-_]?val(ue)?(?!.*stars)", re.IGNORECASE)
+    matches = []
+
+    if isinstance(data, dict):
+        matches = [key for key in data.keys() if pattern.search(str(key))]
+    elif (
+        isinstance(data, (np.ndarray, list))
+        and len(data) > 0
+        and isinstance(data[0], dict)
+    ):
+        matches = [key for key in data[0].keys() if pattern.search(str(key))]
+
+    return matches if multiple else (matches[0] if matches else None)
+
+
+def _find_pval_col(
+    df: pd.DataFrame, multiple: bool = False
+) -> Union[Optional[str], List[str]]:
+    """
+    Find p-value column name(s) in a DataFrame.
+
+    Example:
+    --------
+    >>> df = pd.DataFrame({'p_value': [0.05, 0.01], 'pval': [0.1, 0.001], 'other': [1, 2]})
+    >>> find_pval_col(df)
+    ['p_value', 'pval']
+    >>> find_pval_col(df, multiple=False)
+    'p_value'
+
+    Parameters:
+    -----------
+    df : pd.DataFrame
+        DataFrame to search for p-value column(s)
+    multiple : bool, optional
+        If True, return all matches; if False, return only the first match (default is False)
+
+    Returns:
+    --------
+    Union[Optional[str], List[str]]
+        Name(s) of the column(s) that match p-value patterns, or None if not found
+    """
+    pattern = re.compile(r"p[-_]?val(ue)?(?!.*stars)", re.IGNORECASE)
+    matches = [col for col in df.columns if pattern.search(str(col))]
+
+    return matches if multiple else (matches[0] if matches else None)
+
+
+# EOF

scitex/pd/_force_df.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-04-27 19:59:11 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/pd/_force_df.py
+# ----------------------------------------
+import os
+
+__FILE__ = "./src/scitex/pd/_force_df.py"
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+import numpy as np
+import pandas as pd
+
+from ..types import is_listed_X
+
+
+def force_df(data, filler=np.nan):
+    """
+    Convert various data types to pandas DataFrame.
+    
+    Parameters
+    ----------
+    data : various
+        The data to convert to DataFrame. Can be DataFrame, Series, ndarray, 
+        list, tuple, dict, scalar value, etc.
+    filler : any, optional
+        Value to use for filling missing values, by default np.nan
+        
+    Returns
+    -------
+    pd.DataFrame
+        Data converted to DataFrame
+        
+    Examples
+    --------
+    >>> import scitex
+    >>> import pandas as pd
+    >>> import numpy as np
+    
+    # DataFrame input returns the same DataFrame
+    >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
+    >>> scitex.pd.force_df(df) is df
+    True
+    
+    # Series input is converted to DataFrame
+    >>> series = pd.Series([1, 2, 3], name='test')
+    >>> scitex.pd.force_df(series)
+       test
+    0     1
+    1     2
+    2     3
+    
+    # NumPy array input is converted to DataFrame
+    >>> arr = np.array([1, 2, 3])
+    >>> scitex.pd.force_df(arr)
+       value
+    0      1
+    1      2
+    2      3
+    
+    # Scalar values are converted to single-value DataFrames
+    >>> scitex.pd.force_df(42)
+       value
+    0     42
+    
+    # Lists and tuples are converted to DataFrame
+    >>> scitex.pd.force_df([1, 2, 3])
+       value
+    0      1
+    1      2
+    2      3
+    
+    # Dictionaries are converted to DataFrame with appropriate handling
+    # of different length values
+    >>> data = {'A': [1, 2, 3], 'B': [4, 5]}
+    >>> scitex.pd.force_df(data)
+       A  B
+    0  1  4
+    1  2  5
+    2  3  NaN
+    """
+    # Return None as empty DataFrame
+    if data is None:
+        return pd.DataFrame()
+        
+    # Return DataFrame as is
+    if isinstance(data, pd.DataFrame):
+        return data
+        
+    # Convert Series to DataFrame
+    if isinstance(data, pd.Series):
+        return data.to_frame()
+        
+    # Convert numpy array to DataFrame
+    if isinstance(data, np.ndarray):
+        # Handle 1D array
+        if data.ndim == 1:
+            return pd.DataFrame(data, columns=['value'])
+        # Handle 2D array
+        elif data.ndim == 2:
+            return pd.DataFrame(data)
+        # Handle higher dimensional arrays
+        else:
+            shape = data.shape
+            reshaped = data.reshape(shape[0], -1)
+            return pd.DataFrame(reshaped)
+            
+    # Handle scalar values (int, float, str, etc.)
+    if isinstance(data, (int, float, str, bool)):
+        return pd.DataFrame([data], columns=['value'])
+            
+    # Handle lists and tuples
+    if isinstance(data, (list, tuple)):
+        # Handle list of lists/arrays -> DataFrame
+        if len(data) > 0 and isinstance(data[0], (list, tuple, np.ndarray)):
+            return pd.DataFrame(data)
+        # Handle simple list/tuple -> single column DataFrame
+        else:
+            return pd.DataFrame(data, columns=['value'])
+    
+    # Continue with the original implementation for dictionaries
+    if isinstance(data, dict):
+        # Original implementation
+        permutable_dict = data.copy()
+        
+        # Get the lengths
+        max_len = 0
+        for k, v in permutable_dict.items():
+            # Check if v is an iterable (but not string) or treat as single length otherwise
+            if isinstance(v, (str, int, float)) or not hasattr(v, "__len__"):
+                length = 1
+            else:
+                length = len(v)
+            max_len = max(max_len, length)
+        
+        # Replace with appropriately filled list
+        for k, v in permutable_dict.items():
+            if isinstance(v, (str, int, float)) or not hasattr(v, "__len__"):
+                permutable_dict[k] = [v] + [filler] * (max_len - 1)
+            else:
+                permutable_dict[k] = list(v) + [filler] * (max_len - len(v))
+        
+        # Puts them into a DataFrame
+        return pd.DataFrame(permutable_dict)
+    
+    # For any other iterable type
+    try:
+        return pd.DataFrame(list(data), columns=['value'])
+    except:
+        raise TypeError(f"Cannot convert object of type {type(data)} to DataFrame")
+
+
+# EOF

scitex/pd/_from_xyz.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-09-26 07:22:18 (ywatanabe)"
+# /home/ywatanabe/proj/_scitex_repo_openhands/src/scitex/pd/_from_xyz.py
+
+import pandas as pd
+import numpy as np
+
+
+def from_xyz(data_frame, x=None, y=None, z=None, square=False):
+    """
+    Convert a DataFrame with 'x', 'y', 'z' format into a heatmap DataFrame.
+
+    Example
+    -------
+    import pandas as pd
+    data = pd.DataFrame({
+        'col1': ['A', 'B', 'C', 'A'],
+        'col2': ['X', 'Y', 'Z', 'Y'],
+        'p_val': [0.01, 0.05, 0.001, 0.1]
+    })
+    data = data.rename(columns={"col1": "x", "col2": "y", "p_val": "z"})
+    result = from_xyz(data)
+    print(result)
+
+    Parameters
+    ----------
+    data_frame : pandas.DataFrame
+        Input DataFrame with columns for x, y, and z values.
+    x : str, optional
+        Name of the column to use as x-axis. Defaults to 'x'.
+    y : str, optional
+        Name of the column to use as y-axis. Defaults to 'y'.
+    z : str, optional
+        Name of the column to use as z-values. Defaults to 'z'.
+    square : bool, optional
+        If True, force the output to be a square matrix. Defaults to False.
+
+    Returns
+    -------
+    pandas.DataFrame
+        A DataFrame in heatmap/pivot format.
+    """
+    x = x or "x"
+    y = y or "y"
+    z = z or "z"
+
+    heatmap = pd.pivot_table(data_frame, values=z, index=y, columns=x, aggfunc="first")
+
+    if square:
+        # Make it square by including all unique labels
+        all_labels = sorted(set(heatmap.index) | set(heatmap.columns))
+        heatmap = heatmap.reindex(index=all_labels, columns=all_labels)
+
+    heatmap = heatmap.fillna(0)
+
+    return heatmap
+
+
+if __name__ == "__main__":
+    np.random.seed(42)
+    stats = pd.DataFrame(
+        {
+            "col1": np.random.choice(["A", "B", "C"], 100),
+            "col2": np.random.choice(["X", "Y", "Z"], 100),
+            "p_val": np.random.rand(100),
+        }
+    )
+    stats = stats.rename(columns={"col1": "x", "col2": "y", "p_val": "z"})
+    result = from_xyz(stats)
+    print(result)

scitex/pd/_ignore_SettingWithCopyWarning.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-05 07:35:30 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/pd/_ignore_.py
+
+import warnings
+from contextlib import contextmanager
+
+
+@contextmanager
+def ignore_setting_with_copy_warning():
+    """
+    Context manager to temporarily ignore pandas SettingWithCopyWarning.
+
+    Example
+    -------
+    >>> with ignore_SettingWithCopyWarning():
+    ...     df['column'] = new_values  # No warning will be shown
+    """
+    try:
+        from pandas.errors import SettingWithCopyWarning
+    except ImportError:
+        from pandas.core.common import SettingWithCopyWarning
+
+    # Save current warning filters
+    with warnings.catch_warnings():
+        warnings.simplefilter(action="ignore", category=SettingWithCopyWarning)
+        yield
+
+
+# Backward compatibility
+ignore_SettingWithCopyWarning = ignore_setting_with_copy_warning  # Deprecated
+
+# EOF

scitex/pd/_melt_cols.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-10-05 23:04:16 (ywatanabe)"
+# /home/ywatanabe/proj/_scitex_repo_openhands/src/scitex/pd/_melt_cols.py
+
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-10-05 23:03:39 (ywatanabe)"
+# /home/ywatanabe/proj/_scitex_repo_openhands/src/scitex/pd/_melt_cols.py
+
+from typing import List, Optional
+import pandas as pd
+
+
+def melt_cols(
+    df: pd.DataFrame, cols: List[str], id_columns: Optional[List[str]] = None
+) -> pd.DataFrame:
+    """
+    Melt specified columns while preserving links to other data in a DataFrame.
+
+    Example
+    -------
+    >>> data = pd.DataFrame({
+    ...     'id': [1, 2, 3],
+    ...     'name': ['Alice', 'Bob', 'Charlie'],
+    ...     'score_1': [85, 90, 78],
+    ...     'score_2': [92, 88, 95]
+    ... })
+    >>> melted = melt_cols(data, cols=['score_1', 'score_2'])
+    >>> print(melted)
+       id     name variable  value
+    0   1    Alice  score_1     85
+    1   2      Bob  score_1     90
+    2   3  Charlie  score_1     78
+    3   1    Alice  score_2     92
+    4   2      Bob  score_2     88
+    5   3  Charlie  score_2     95
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Input DataFrame
+    cols : List[str]
+        Columns to be melted
+    id_columns : Optional[List[str]], default None
+        Columns to preserve as identifiers. If None, all columns not in 'cols' are used.
+
+    Returns
+    -------
+    pd.DataFrame
+        Melted DataFrame with preserved identifier columns
+
+    Raises
+    ------
+    ValueError
+        If cols are not present in the DataFrame
+    """
+    missing_melt = set(cols) - set(df.columns)
+    if missing_melt:
+        raise ValueError(f"Columns not found in DataFrame: {missing_melt}")
+
+    if id_columns is None:
+        id_columns = [col for col in df.columns if col not in cols]
+
+    df_copy = df.reset_index(drop=True)
+    df_copy["global_index"] = df_copy.index
+
+    # Use a different value_name if "value" is one of the columns being melted
+    value_name = "value" if "value" not in cols else "melted_value"
+    melted_df = df_copy[cols + ["global_index"]].melt(
+        id_vars=["global_index"], value_name=value_name
+    )
+    if id_columns:
+        formatted_df = melted_df.merge(
+            df_copy[id_columns + ["global_index"]], on="global_index"
+        )
+        return formatted_df.drop("global_index", axis=1)
+    else:
+        # No id columns to merge, just return melted data without global_index
+        return melted_df.drop("global_index", axis=1)