py2ls 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

py2ls/ips.py

@@ -793,6 +793,99 @@ def str2html(text_list, strict=False):
     return html_content
 
 
+def cm2px(*cm, dpi=300) -> list:
+    # Case 1: When the user passes a single argument that is a list or tuple, such as cm2px([8, 5]) or inch2cm((8, 5))
+    if len(cm) == 1 and isinstance(cm[0], (list, tuple)):
+        # If the input is a single list or tuple, we unpack its elements and convert each to cm
+        return [i / 2.54 * dpi for i in cm[0]]
+    # Case 2: When the user passes multiple arguments directly, such as cm2px(8, 5)
+    else:
+        return [i / 2.54 * dpi for i in cm]
+
+
+def px2cm(*px, dpi=300) -> list:
+    # Case 1: When the user passes a single argument that is a list or tuple, such as px2cm([8, 5]) or inch2cm((8, 5))
+    if len(px) == 1 and isinstance(px[0], (list, tuple)):
+        # If the input is a single list or tuple, we unpack its elements and convert each to cm
+        return [i * 2.54 / dpi for i in px[0]]
+    # Case 2: When the user passes multiple arguments directly, such as px2cm(8, 5)
+    else:
+        # Here, we convert each individual argument directly to cm
+        return [i * 2.54 / dpi for i in px]
+
+
+def px2inch(*px, dpi=300) -> list:
+    """
+    px2inch: converts pixel measurements to inches based on the given dpi.
+    Usage:
+    px2inch(300, 600, dpi=300); px2inch([300, 600], dpi=300)
+    Returns:
+        list: in inches
+    """
+    # Case 1: When the user passes a single argument that is a list or tuple, such as px2inch([300, 600]) or px2inch((300, 600))
+    if len(px) == 1 and isinstance(px[0], (list, tuple)):
+        # If the input is a single list or tuple, we unpack its elements and convert each to inches
+        return [i / dpi for i in px[0]]
+    # Case 2: When the user passes multiple arguments directly, such as px2inch(300, 600)
+    else:
+        # Here, we convert each individual argument directly to inches
+        return [i / dpi for i in px]
+
+
+def cm2inch(*cm) -> list:
+    """
+    cm2inch: converts centimeter measurements to inches.
+    Usage:
+    cm2inch(10, 12.7); cm2inch((10, 12.7)); cm2inch([10, 12.7])
+    Returns:
+        list: in inches
+    """
+    # Case 1: When the user passes a single argument that is a list or tuple, such as cm2inch([10, 12.7]) or cm2inch((10, 12.7))
+    if len(cm) == 1 and isinstance(cm[0], (list, tuple)):
+        # If the input is a single list or tuple, we unpack its elements and convert each to inches
+        return [i * 2.54 for i in cm[0]]
+    # Case 2: When the user passes multiple arguments directly, such as cm2inch(10, 12.7)
+    else:
+        # Here, we convert each individual argument directly to inches
+        return [i * 2.54 for i in cm]
+
+
+def inch2px(*inch, dpi=300) -> list:
+    """
+    inch2px: converts inch measurements to pixels based on the given dpi.
+    Usage:
+    inch2px(1, 2, dpi=300); inch2px([1, 2], dpi=300)
+    Returns:
+        list: in pixels
+    """
+    # Case 1: When the user passes a single argument that is a list or tuple, such as inch2px([1, 2]) or inch2px((1, 2))
+    if len(inch) == 1 and isinstance(inch[0], (list, tuple)):
+        # If the input is a single list or tuple, we unpack its elements and convert each to pixels
+        return [i * dpi for i in inch[0]]
+    # Case 2: When the user passes multiple arguments directly, such as inch2px(1, 2)
+    else:
+        # Here, we convert each individual argument directly to pixels
+        return [i * dpi for i in inch]
+
+
+def inch2cm(*inch) -> list:
+    """
+    inch2cm: converts inch measurements to centimeters.
+    Usage:
+    inch2cm(8,5); inch2cm((8,5)); inch2cm([8,5])
+    Returns:
+        list: in centimeters
+    """
+    # Case 1: When the user passes a single argument that is a list or tuple, such as inch2cm([8, 5]) or inch2cm((8, 5))
+    if len(inch) == 1 and isinstance(inch[0], (list, tuple)):
+        # If the input is a single list or tuple, we unpack its elements and convert each to cm
+        return [i / 2.54 for i in inch[0]]
+    # Case 2: When the user passes multiple arguments directly, such as inch2cm(8, 5)
+    else:
+        # Here, we convert each individual argument directly to cm
+        return [i / 2.54 for i in inch]
+
+
 def sreplace(*args, **kwargs):
     """
     sreplace(text, by=None, robust=True)
@@ -1363,6 +1456,80 @@ def unzip(dir_path, output_dir=None):
 # output_dir_7z = unzip('archive.7z')
 
 
+def is_df_abnormal(df: pd.DataFrame, verbose=False) -> bool:
+    """
+    Usage
+    is_abnormal = is_df_abnormal(df, verbose=1)
+
+    """
+    # Initialize a list to hold messages about abnormalities
+    messages = []
+    is_abnormal = False
+    # Check the shape of the DataFrame
+    actual_shape = df.shape
+    messages.append(f"Shape of DataFrame: {actual_shape}")
+
+    # Check column names
+    column_names = df.columns.tolist()
+
+    # Count of delimiters and their occurrences
+    delimiter_counts = {"\t": 0, ",": 0, "\n": 0, "": 0}  # Count of empty strings
+
+    for name in column_names:
+        # Count occurrences of each delimiter
+        delimiter_counts["\t"] += name.count("\t")
+        delimiter_counts[","] += name.count(",")
+        delimiter_counts["\n"] += name.count("\n")
+        if name.strip() == "":
+            delimiter_counts[""] += 1
+
+    # Check for abnormalities based on delimiter counts
+    if len(column_names) == 1 and delimiter_counts["\t"] > 1:
+        messages.append("Abnormal: Column names are not split correctly.")
+        is_abnormal = True
+
+    if any(delimiter_counts[d] > 3 for d in delimiter_counts if d != ""):
+        messages.append("Abnormal: Too many delimiters in column names.")
+        is_abnormal = True
+
+    if delimiter_counts[""] > 3:
+        messages.append("Abnormal: There are empty column names.")
+        is_abnormal = True
+
+    if any(delimiter_counts[d] > 3 for d in ["\t", ",", "\n"]):
+        messages.append("Abnormal: Some column names contain unexpected characters.")
+        is_abnormal = True
+
+    # Check for missing values
+    missing_values = df.isnull().sum()
+    if missing_values.any():
+        messages.append("Missing values in columns:")
+        messages.append(missing_values[missing_values > 0].to_string())
+        is_abnormal = True
+
+    # Check data types
+    data_types = df.dtypes
+    # messages.append(f"Data types of columns:\n{data_types}")
+
+    # Check for constant values across any column
+    constant_columns = df.columns[df.nunique() == 1].tolist()
+    if constant_columns:
+        messages.append(f"Abnormal: Columns with constant values: {constant_columns}")
+        is_abnormal = True
+
+    # Check for an unreasonable number of rows or columns
+    if actual_shape[0] < 2 or actual_shape[1] < 2:
+        messages.append(
+            "Abnormal: DataFrame is too small (less than 2 rows or columns)."
+        )
+        is_abnormal = True
+
+    # Compile results
+    if verbose:
+        print("\n".join(messages))
+    return is_abnormal  # Data is abnormal
+
+
 def fload(fpath, kind=None, **kwargs):
     """
     Load content from a file with specified file type.
@@ -1399,9 +1566,43 @@ def fload(fpath, kind=None, **kwargs):
         root = tree.getroot()
         return etree.tostring(root, pretty_print=True).decode()
 
+    def get_comment(fpath, comment=None, encoding="utf-8", lines_to_check=5):
+        """
+        Detect comment characters in a file.
+
+        Parameters:
+        - fpath: str, the file path of the CSV file.
+        - encoding: str, the encoding of the file (default is 'utf-8').
+        - lines_to_check: int, number of lines to check for comment characters (default is 5).
+
+        Returns:
+        - str or None: the detected comment character, or None if no comment character is found.
+        """
+        comment_chars = [
+            "#",
+            "!",
+            "//",
+            ";",
+        ]  # can use any character or string as a comment
+        try:
+            with open(fpath, "r", encoding=encoding) as f:
+                lines = [next(f) for _ in range(lines_to_check)]
+        except (UnicodeDecodeError, ValueError):
+            with open(fpath, "r", encoding=get_encoding(fpath)) as f:
+                lines = [next(f) for _ in range(lines_to_check)]
+        for line in lines:
+            for char in comment_chars:
+                if line.startswith(char):
+                    return char
+        return None
+
     def load_csv(fpath, **kwargs):
+        from pandas.errors import EmptyDataError
+
         engine = kwargs.get("engine", "pyarrow")
         kwargs.pop("engine", None)
+        sep = kwargs.get("sep", "\t")
+        kwargs.pop("sep", None)
         index_col = kwargs.get("index_col", None)
         kwargs.pop("index_col", None)
         memory_map = kwargs.get("memory_map", True)
@@ -1410,48 +1611,134 @@ def fload(fpath, kind=None, **kwargs):
         kwargs.pop("skipinitialspace", None)
         encoding = kwargs.get("encoding", "utf-8")
         kwargs.pop("encoding", None)
+        on_bad_lines = kwargs.get("on_bad_lines", "skip")
+        kwargs.pop("on_bad_lines", None)
+        comment = kwargs.get("comment", None)
+        kwargs.pop("comment", None)
+
+        if comment is None:
+            comment = get_comment(
+                fpath, comment=None, encoding="utf-8", lines_to_check=5
+            )
+
         try:
-            if engine == "pyarrow":
-                df = pd.read_csv(
-                    fpath,
-                    engine=engine,
-                    index_col=index_col,
-                    encoding=encoding,
-                    **kwargs,
-                )
-            else:
-                df = pd.read_csv(
-                    fpath,
-                    engine=engine,
-                    index_col=index_col,
-                    memory_map=memory_map,
-                    encoding=encoding,
-                    skipinitialspace=skipinitialspace,
-                    **kwargs,
-                )
-            print("File loaded successfully with utf-8 encoding.")
-        except UnicodeDecodeError:
-            encoding = get_encoding(fpath)
-            print(f"utf-8 failed. Retrying with detected encoding: {encoding}")
-            if engine == "pyarrow":
-                df = pd.read_csv(
-                    fpath,
-                    engine=engine,
-                    index_col=index_col,
-                    encoding=encoding,
-                    **kwargs,
-                )
-            else:
-                df = pd.read_csv(
-                    fpath,
-                    engine=engine,
-                    index_col=index_col,
-                    memory_map=memory_map,
-                    encoding=encoding,
-                    skipinitialspace=skipinitialspace,
-                    **kwargs,
-                )
-            print("File loaded successfully with utf-8 encoding.")
+            df = pd.read_csv(
+                fpath,
+                engine=engine,
+                index_col=index_col,
+                memory_map=memory_map,
+                encoding=encoding,
+                comment=comment,
+                skipinitialspace=skipinitialspace,
+                sep=sep,
+                on_bad_lines=on_bad_lines,
+                **kwargs,
+            )
+        except:
+            try:
+                try:
+                    if engine == "pyarrow":
+                        df = pd.read_csv(
+                            fpath,
+                            engine=engine,
+                            index_col=index_col,
+                            encoding=encoding,
+                            sep=sep,
+                            on_bad_lines=on_bad_lines,
+                            comment=comment,
+                            **kwargs,
+                        )
+                    else:
+                        df = pd.read_csv(
+                            fpath,
+                            engine=engine,
+                            index_col=index_col,
+                            memory_map=memory_map,
+                            encoding=encoding,
+                            sep=sep,
+                            skipinitialspace=skipinitialspace,
+                            on_bad_lines=on_bad_lines,
+                            comment=comment,
+                            **kwargs,
+                        )
+
+                    if is_df_abnormal(df, verbose=0):
+                        raise ValueError("the df is abnormal")
+                except (UnicodeDecodeError, ValueError):
+                    encoding = get_encoding(fpath)
+                    # print(f"utf-8 failed. Retrying with detected encoding: {encoding}")
+                    if engine == "pyarrow":
+                        df = pd.read_csv(
+                            fpath,
+                            engine=engine,
+                            index_col=index_col,
+                            encoding=encoding,
+                            sep=sep,
+                            on_bad_lines=on_bad_lines,
+                            comment=comment,
+                            **kwargs,
+                        )
+                    else:
+                        df = pd.read_csv(
+                            fpath,
+                            engine=engine,
+                            index_col=index_col,
+                            memory_map=memory_map,
+                            encoding=encoding,
+                            sep=sep,
+                            skipinitialspace=skipinitialspace,
+                            on_bad_lines=on_bad_lines,
+                            comment=comment,
+                            **kwargs,
+                        )
+                    if is_df_abnormal(df, verbose=0):
+                        raise ValueError("the df is abnormal")
+            except Exception as e:
+                separators = [",", "\t", ";", "|", " "]
+                for sep in separators:
+                    sep2show = sep if sep != "\t" else "\\t"
+                    # print(f'trying with: engine=pyarrow, sep="{sep2show}"')
+                    try:
+                        df = pd.read_csv(
+                            fpath,
+                            engine=engine,
+                            skipinitialspace=skipinitialspace,
+                            sep=sep,
+                            on_bad_lines=on_bad_lines,
+                            comment=comment,
+                            **kwargs,
+                        )
+                        if not is_df_abnormal(df, verbose=0):  # normal
+                            break
+                        else:
+                            if is_df_abnormal(df, verbose=0):
+                                pass
+                    except:
+                        pass
+                else:
+                    engines = ["c", "python"]
+                    for engine in engines:
+                        # separators = [",", "\t", ";", "|", " "]
+                        for sep in separators:
+                            try:
+                                sep2show = sep if sep != "\t" else "\\t"
+                                # print(f"trying with: engine={engine}, sep='{sep2show}'")
+                                df = pd.read_csv(
+                                    fpath,
+                                    engine=engine,
+                                    sep=sep,
+                                    on_bad_lines=on_bad_lines,
+                                    comment=comment,
+                                    **kwargs,
+                                )
+                                if not is_df_abnormal(df, verbose=0):
+                                    break
+                            except EmptyDataError as e:
+                                continue
+                        else:
+                            pass
+        display(df.head(2))
+        print(f"shape: {df.shape}")
         return df
 
     def load_xlsx(fpath, **kwargs):
@@ -1575,9 +1862,12 @@ def fload(fpath, kind=None, **kwargs):
         "rar",
         "tgz",
     ]
-    supported_types = [*doc_types, *img_types, *zip_types]
+    other_types = ["fcs"]
+    supported_types = [*doc_types, *img_types, *zip_types, *other_types]
     if kind not in supported_types:
-        print(f'Error:\n"{kind}" is not in the supported list {supported_types}')
+        print(
+            f'Warning:\n"{kind}" is not in the supported list '
+        )  # {supported_types}')
     # if os.path.splitext(fpath)[1][1:].lower() in zip_types:
     #     keep=kwargs.get("keep", False)
     #     ifile=kwargs.get("ifile",(0,0))
@@ -1607,7 +1897,8 @@ def fload(fpath, kind=None, **kwargs):
     elif kind == "xml":
         return load_xml(fpath)
     elif kind == "csv":
-        return load_csv(fpath, **kwargs)
+        content = load_csv(fpath, **kwargs)
+        return content
     elif kind in ["ods", "ods", "odt"]:
         engine = kwargs.get("engine", "odf")
         kwargs.pop("engine", None)
@@ -1615,9 +1906,13 @@ def fload(fpath, kind=None, **kwargs):
     elif kind == "xls":
         engine = kwargs.get("engine", "xlrd")
         kwargs.pop("engine", None)
-        return load_xlsx(fpath, engine=engine, **kwargs)
+        content = load_xlsx(fpath, engine=engine, **kwargs)
+        display(content.head(2))
+        return content
     elif kind == "xlsx":
-        return load_xlsx(fpath, **kwargs)
+        content = load_xlsx(fpath, **kwargs)
+        display(content.head(2))
+        return content
     elif kind == "ipynb":
         return load_ipynb(fpath, **kwargs)
     elif kind == "pdf":
@@ -1653,7 +1948,18 @@ def fload(fpath, kind=None, **kwargs):
 
         gene_sets = gp.read_gmt(fpath)
         return gene_sets
+
+    elif kind.lower() == "fcs":
+        import fcsparser
+
+        # https://github.com/eyurtsev/fcsparser
+        meta, data = fcsparser.parse(fpath, reformat_meta=True)
+        return meta, data
+
     else:
+        # try:
+        #     content = load_csv(fpath, **kwargs)
+        # except:
         try:
             try:
                 with open(fpath, "r", encoding="utf-8") as f:
@@ -2126,13 +2432,23 @@ def get_os():
 
 def listdir(
     rootdir,
-    kind="folder",
+    kind=None,
     sort_by="name",
     ascending=True,
     contains=None,
     orient="list",
     output="df",  # 'list','dict','records','index','series'
+    verbose=True,
 ):
+    if kind is None:
+        df_all = pd.DataFrame(
+            {
+                "fname": os.listdir(rootdir),
+                "fpath": [os.path.join(rootdir, i) for i in os.listdir(rootdir)],
+            }
+        )
+        display(df_all)
+        return df_all
     if isinstance(kind, list):
         f_ = []
         for kind_ in kind:
@@ -2204,7 +2520,7 @@ def listdir(
 
         f["num"] = i
         f["rootdir"] = rootdir
-        f["os"] = os.uname().machine
+        f["os"] = get_os()  # os.uname().machine
     else:
         raise FileNotFoundError(
             'The directory "{}" does NOT exist. Please check the directory "rootdir".'.format(
@@ -2227,6 +2543,8 @@ def listdir(
         f = sort_kind(f, by="size", ascending=ascending)
 
     if "df" in output:
+        if verbose:
+            display(f.head())
         return f
     else:
         if "l" in orient.lower():  # list # default
@@ -2259,32 +2577,54 @@ def func_list(lib_name, opt="call"):
     return list_func(lib_name, opt=opt)
 
 
-def mkdir(*args, **kwargs):
+def mkdir_nest(fpath: str) -> str:
     """
-    newfolder(pardir, chdir)
-    Args:
-        pardir (dir): parent dir
-        chdir (str): children dir
-        overwrite (bool): overwrite?
+    Create nested directories based on the provided file path.
+
+    Parameters:
+    - fpath (str): The full file path for which the directories should be created.
+
     Returns:
-        mkdir, giving a option if exists_ok or not
+    - str: The path of the created directory.
     """
-    overwrite = kwargs.get("overwrite", False)
-    for arg in args:
-        if isinstance(arg, (str, list)):
-            if "/" in arg or "\\" in arg:
-                pardir = arg
-                print(f'pardir: "{pardir}"')
-            else:
-                chdir = arg
-                print(f'chdir:"{chdir}"')
-        elif isinstance(arg, bool):
-            overwrite = arg
-            print(overwrite)
-        else:
-            print(f"{arg}Error: not support a {type(arg)} type")
+    # Check if the directory already exists
+    if os.path.isdir(fpath):
+        return fpath
+
+    # Split the full path into directories
+    f_slash = "/" if "mac" in get_os().lower() else "\\"
+    dir_parts = fpath.split(f_slash)  # Split the path by the OS-specific separator
+
+    # Start creating directories from the root to the desired path
+    current_path = ""
+    for part in dir_parts:
+        if part:
+            current_path = os.path.join(current_path, part)
+            if not os.path.isdir(current_path):
+                os.makedirs(current_path)
+    if not current_path.endswith(f_slash):
+        current_path += f_slash
+    return current_path
+
+
+def mkdir(pardir: str = None, chdir: str | list = None, overwrite=False):
+    """
+    Create a directory.
+
+    Parameters:
+    - pardir (str): Parent directory where the new directory will be created. If None, uses the current working directory.
+    - chdir (str | list): Name of the new directory or a list of directories to create.
+                          If None, a default name 'new_directory' will be used.
+    - overwrite (bool): If True, overwrite the directory if it already exists. Defaults to False.
+
+    Returns:
+    - str: The path of the created directory or an error message.
+    """
+
     rootdir = []
     # Convert string to list
+    if chdir is None:
+        return mkdir_nest(pardir)
     if isinstance(chdir, str):
         chdir = [chdir]
     # Subfoldername should be unique
@@ -2331,54 +2671,111 @@ def mkdir(*args, **kwargs):
     return rootdir
 
 
+def split_path(fpath):
+    f_slash = "/" if "mac" in get_os().lower() else "\\"
+    dir_par = f_slash.join(fpath.split(f_slash)[:-1])
+    dir_ch = "".join(fpath.split(f_slash)[-1:])
+    return dir_par, dir_ch
+
+
 def figsave(*args, dpi=300):
     dir_save = None
     fname = None
+    img = None
     for arg in args:
         if isinstance(arg, str):
             if "/" in arg or "\\" in arg:
                 dir_save = arg
             elif "/" not in arg and "\\" not in arg:
                 fname = arg
-    # Backup original values
-    if "/" in dir_save:
-        if dir_save[-1] != "/":
-            dir_save = dir_save + "/"
-    elif "\\" in dir_save:
-        if dir_save[-1] != "\\":
-            dir_save = dir_save + "\\"
-    else:
-        raise ValueError("Check the Path of dir_save Directory")
+        elif isinstance(arg, (Image.Image, np.ndarray)):
+            img = arg  # Store the PIL image if provided
+
+    f_slash = "/" if "mac" in get_os().lower() else "\\"
+    dir_par = f_slash.join(dir_save.split(f_slash)[:-1])
+    dir_ch = "".join(dir_save.split(f_slash)[-1:])
+    if not dir_par.endswith(f_slash):
+        dir_par += f_slash
+    if fname is None:
+        fname = dir_ch
+    mkdir(dir_par)
     ftype = fname.split(".")[-1]
     if len(fname.split(".")) == 1:
         ftype = "nofmt"
-        fname = dir_save + fname + "." + ftype
+        fname = dir_par + fname + "." + ftype
     else:
-        fname = dir_save + fname
+        fname = dir_par + fname
+
     # Save figure based on file type
     if ftype.lower() == "eps":
         plt.savefig(fname, format="eps", bbox_inches="tight")
         plt.savefig(
-            fname.replace(".eps", ".pdf"), format="pdf", bbox_inches="tight", dpi=dpi
+            fname.replace(".eps", ".pdf"),
+            format="pdf",
+            bbox_inches="tight",
+            dpi=dpi,
+            pad_inches=0,
         )
     elif ftype.lower() == "nofmt":  # default: both "tif" and "pdf"
         fname_corr = fname.replace("nofmt", "pdf")
-        plt.savefig(fname_corr, format="pdf", bbox_inches="tight", dpi=dpi)
+        plt.savefig(
+            fname_corr, format="pdf", bbox_inches="tight", dpi=dpi, pad_inches=0
+        )
         fname = fname.replace("nofmt", "tif")
-        plt.savefig(fname, format="tiff", dpi=dpi, bbox_inches="tight")
+        plt.savefig(fname, format="tiff", dpi=dpi, bbox_inches="tight", pad_inches=0)
         print(f"default saving filetype: both 'tif' and 'pdf")
     elif ftype.lower() == "pdf":
-        plt.savefig(fname, format="pdf", bbox_inches="tight", dpi=dpi)
-    elif ftype.lower() in ["jpg", "jpeg"]:
-        plt.savefig(fname, format="jpeg", dpi=dpi, bbox_inches="tight")
-    elif ftype.lower() == "png":
-        plt.savefig(fname, format="png", dpi=dpi, bbox_inches="tight", transparent=True)
-    elif ftype.lower() in ["tiff", "tif"]:
-        plt.savefig(fname, format="tiff", dpi=dpi, bbox_inches="tight")
+        plt.savefig(fname, format="pdf", bbox_inches="tight", dpi=dpi, pad_inches=0)
+    elif ftype.lower() in ["jpg", "jpeg", "png", "tiff", "tif"]:
+        if img is not None:  # If a PIL image is provided
+            if isinstance(img, Image.Image):
+                if img.mode == "RGBA":
+                    img = img.convert("RGB")
+                img.save(fname, format=ftype.upper(), dpi=(dpi, dpi))
+            elif isinstance(img, np.ndarray):
+                import cv2
+
+                # Check the shape of the image to determine color mode
+                if img.ndim == 2:
+                    # Grayscale image
+                    Image.fromarray(img).save(
+                        fname, format=ftype.upper(), dpi=(dpi, dpi)
+                    )
+                elif img.ndim == 3:
+                    if img.shape[2] == 3:
+                        # RGB image
+                        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+                        Image.fromarray(img).save(
+                            fname, format=ftype.upper(), dpi=(dpi, dpi)
+                        )
+                    elif img.shape[2] == 4:
+                        # RGBA image
+                        img = cv2.cvtColor(
+                            img, cv2.COLOR_BGRA2RGBA
+                        )  # Convert BGRA to RGBA
+                        Image.fromarray(img).save(
+                            fname, format=ftype.upper(), dpi=(dpi, dpi)
+                        )
+                    else:
+                        raise ValueError(
+                            "Unexpected number of channels in the image array."
+                        )
+                else:
+                    raise ValueError(
+                        "Image array has an unexpected number of dimensions."
+                    )
+        else:
+            plt.savefig(
+                fname, format=ftype.lower(), dpi=dpi, bbox_inches="tight", pad_inches=0
+            )
+    # elif ftype.lower() == "png":
+    #     plt.savefig(fname, format="png", dpi=dpi, bbox_inches="tight", transparent=True,pad_inches=0)
+    # elif ftype.lower() in ["tiff", "tif"]:
+    #     plt.savefig(fname, format="tiff", dpi=dpi, bbox_inches="tight",pad_inches=0)
     elif ftype.lower() == "emf":
-        plt.savefig(fname, format="emf", dpi=dpi, bbox_inches="tight")
+        plt.savefig(fname, format="emf", dpi=dpi, bbox_inches="tight", pad_inches=0)
     elif ftype.lower() == "fig":
-        plt.savefig(fname, format="pdf", bbox_inches="tight", dpi=dpi)
+        plt.savefig(fname, format="pdf", bbox_inches="tight", dpi=dpi, pad_inches=0)
     print(f"\nSaved @: dpi={dpi}\n{fname}")
 
 
@@ -2510,6 +2907,8 @@ def load_img(fpath):
         FileNotFoundError: If the specified file is not found.
         OSError: If the specified file cannot be opened or is not a valid image file.
     """
+    from PIL import Image
+
     try:
         img = Image.open(fpath)
         return img
@@ -2650,393 +3049,6 @@ def apply_filter(img, *args):
         return img.filter(supported_filters[filter_name])
 
 
-def imgsetss(
-    img,
-    sets=None,
-    show=True,
-    show_axis=False,
-    size=None,
-    dpi=100,
-    figsize=None,
-    auto=False,
-    filter_kws=None,
-):
-    """
-    Apply various enhancements and filters to an image using PIL's ImageEnhance and ImageFilter modules.
-
-    Args:
-        img (PIL.Image): The input image.
-        sets (dict): A dictionary specifying the enhancements, filters, and their parameters.
-        show (bool): Whether to display the enhanced image.
-        show_axis (bool): Whether to display axes on the image plot.
-        size (tuple): The size of the thumbnail, cover, contain, or fit operation.
-        dpi (int): Dots per inch for the displayed image.
-        figsize (tuple): The size of the figure for displaying the image.
-        auto (bool): Whether to automatically enhance the image based on its characteristics.
-
-    Returns:
-        PIL.Image: The enhanced image.
-
-    Supported enhancements and filters:
-        - "sharpness": Adjusts the sharpness of the image. Values > 1 increase sharpness, while values < 1 decrease sharpness.
-        - "contrast": Adjusts the contrast of the image. Values > 1 increase contrast, while values < 1 decrease contrast.
-        - "brightness": Adjusts the brightness of the image. Values > 1 increase brightness, while values < 1 decrease brightness.
-        - "color": Adjusts the color saturation of the image. Values > 1 increase saturation, while values < 1 decrease saturation.
-        - "rotate": Rotates the image by the specified angle.
-        - "crop" or "cut": Crops the image. The value should be a tuple specifying the crop box as (left, upper, right, lower).
-        - "size": Resizes the image to the specified dimensions.
-        - "thumbnail": Resizes the image to fit within the given size while preserving aspect ratio.
-        - "cover": Resizes and crops the image to fill the specified size.
-        - "contain": Resizes the image to fit within the specified size, adding borders if necessary.
-        - "fit": Resizes and pads the image to fit within the specified size.
-        - "filter": Applies various filters to the image (e.g., BLUR, CONTOUR, EDGE_ENHANCE).
-
-    Note:
-        The "color" and "enhance" enhancements are not implemented in this function.
-    """
-    supported_filters = [
-        "BLUR",
-        "CONTOUR",
-        "DETAIL",
-        "EDGE_ENHANCE",
-        "EDGE_ENHANCE_MORE",
-        "EMBOSS",
-        "FIND_EDGES",
-        "SHARPEN",
-        "SMOOTH",
-        "SMOOTH_MORE",
-        "MIN_FILTER",
-        "MAX_FILTER",
-        "MODE_FILTER",
-        "MULTIBAND_FILTER",
-        "GAUSSIAN_BLUR",
-        "BOX_BLUR",
-        "MEDIAN_FILTER",
-    ]
-    print(
-        "sets: a dict,'sharp:1.2','color','contrast:'auto' or 1.2','bright', 'crop: x_upperleft,y_upperleft, x_lowerright, y_lowerright','rotation','resize','rem or background'"
-    )
-    print(f"usage: filter_kws 'dict' below:")
-    pp([str(i).lower() for i in supported_filters])
-    print("\nlog:\n")
-
-    def confirm_rembg_models(model_name):
-        models_support = [
-            "u2net",
-            "u2netp",
-            "u2net_human_seg",
-            "u2net_cloth_seg",
-            "silueta",
-            "isnet-general-use",
-            "isnet-anime",
-            "sam",
-        ]
-        if model_name in models_support:
-            print(f"model_name: {model_name}")
-            return model_name
-        else:
-            print(
-                f"{model_name} cannot be found, check the name:{models_support}, default('isnet-general-use') has been used"
-            )
-            return "isnet-general-use"
-
-    def auto_enhance(img):
-        """
-        Automatically enhances the image based on its characteristics.
-        Args:
-            img (PIL.Image): The input image.
-        Returns:
-            dict: A dictionary containing the optimal enhancement values.
-        """
-        # Determine the bit depth based on the image mode
-        if img.mode in ["1", "L", "P", "RGB", "YCbCr", "LAB", "HSV"]:
-            # 8-bit depth per channel
-            bit_depth = 8
-        elif img.mode in ["RGBA", "CMYK"]:
-            # 8-bit depth per channel + alpha (RGBA) or additional channels (CMYK)
-            bit_depth = 8
-        elif img.mode in ["I", "F"]:
-            # 16-bit depth per channel (integer or floating-point)
-            bit_depth = 16
-        else:
-            raise ValueError("Unsupported image mode")
-        # Calculate the brightness and contrast for each channel
-        num_channels = len(img.getbands())
-        brightness_factors = []
-        contrast_factors = []
-        for channel in range(num_channels):
-            channel_histogram = img.split()[channel].histogram()
-            brightness = sum(i * w for i, w in enumerate(channel_histogram)) / sum(
-                channel_histogram
-            )
-            channel_min, channel_max = img.split()[channel].getextrema()
-            contrast = channel_max - channel_min
-            # Adjust calculations based on bit depth
-            normalization_factor = 2**bit_depth - 1  # Max value for the given bit depth
-            brightness_factor = (
-                1.0 + (brightness - normalization_factor / 2) / normalization_factor
-            )
-            contrast_factor = (
-                1.0 + (contrast - normalization_factor / 2) / normalization_factor
-            )
-            brightness_factors.append(brightness_factor)
-            contrast_factors.append(contrast_factor)
-        # Calculate the average brightness and contrast factors across channels
-        avg_brightness_factor = sum(brightness_factors) / num_channels
-        avg_contrast_factor = sum(contrast_factors) / num_channels
-        return {"brightness": avg_brightness_factor, "contrast": avg_contrast_factor}
-
-    # Load image if input is a file path
-    if isinstance(img, str):
-        img = load_img(img)
-    img_update = img.copy()
-    # Auto-enhance image if requested
-    if auto:
-        auto_params = auto_enhance(img_update)
-        sets.update(auto_params)
-    if sets is None:
-        sets = {}
-    for k, value in sets.items():
-        if "shar" in k.lower():
-            enhancer = ImageEnhance.Sharpness(img_update)
-            img_update = enhancer.enhance(value)
-        elif "col" in k.lower() and "bg" not in k.lower():
-            enhancer = ImageEnhance.Color(img_update)
-            img_update = enhancer.enhance(value)
-        elif "contr" in k.lower():
-            if value and isinstance(value, (float, int)):
-                enhancer = ImageEnhance.Contrast(img_update)
-                img_update = enhancer.enhance(value)
-            else:
-                print("autocontrasted")
-                img_update = ImageOps.autocontrast(img_update)
-        elif "bri" in k.lower():
-            enhancer = ImageEnhance.Brightness(img_update)
-            img_update = enhancer.enhance(value)
-        elif "cro" in k.lower() or "cut" in k.lower():
-            img_update = img_update.crop(value)
-        elif "rota" in k.lower():
-            img_update = img_update.rotate(value)
-        elif "si" in k.lower():
-            img_update = img_update.resize(value)
-        elif "thum" in k.lower():
-            img_update.thumbnail(value)
-        elif "cover" in k.lower():
-            img_update = ImageOps.cover(img_update, size=value)
-        elif "contain" in k.lower():
-            img_update = ImageOps.contain(img_update, size=value)
-        elif "fit" in k.lower():
-            img_update = ImageOps.fit(img_update, size=value)
-        elif "pad" in k.lower():
-            img_update = ImageOps.pad(img_update, size=value)
-        elif "rem" in k.lower() or "rm" in k.lower() or "back" in k.lower():
-            if value and isinstance(value, (int, float, list)):
-                print(
-                    'example usage: {"rm":[alpha_matting_background_threshold(20),alpha_matting_foreground_threshold(270),alpha_matting_erode_sive(11)]}'
-                )
-                print("https://github.com/danielgatis/rembg/blob/main/USAGE.md")
-                #     ###            Parameters:
-                #         data (Union[bytes, PILImage, np.ndarray]): The input image data.
-                #         alpha_matting (bool, optional): Flag indicating whether to use alpha matting. Defaults to False.
-                #         alpha_matting_foreground_threshold (int, optional): Foreground threshold for alpha matting. Defaults to 240.
-                #         alpha_matting_background_threshold (int, optional): Background threshold for alpha matting. Defaults to 10.
-                #         alpha_matting_erode_size (int, optional): Erosion size for alpha matting. Defaults to 10.
-                #         session (Optional[BaseSession], optional): A session object for the 'u2net' model. Defaults to None.
-                #         only_mask (bool, optional): Flag indicating whether to return only the binary masks. Defaults to False.
-                #         post_process_mask (bool, optional): Flag indicating whether to post-process the masks. Defaults to False.
-                #         bgcolor (Optional[Tuple[int, int, int, int]], optional): Background color for the cutout image. Defaults to None.
-                #  ###
-                if isinstance(value, int):
-                    value = [value]
-                if len(value) < 2:
-                    img_update = remove(
-                        img_update,
-                        alpha_matting=True,
-                        alpha_matting_background_threshold=value,
-                    )
-                elif 2 <= len(value) < 3:
-                    img_update = remove(
-                        img_update,
-                        alpha_matting=True,
-                        alpha_matting_background_threshold=value[0],
-                        alpha_matting_foreground_threshold=value[1],
-                    )
-                elif 3 <= len(value) < 4:
-                    img_update = remove(
-                        img_update,
-                        alpha_matting=True,
-                        alpha_matting_background_threshold=value[0],
-                        alpha_matting_foreground_threshold=value[1],
-                        alpha_matting_erode_size=value[2],
-                    )
-            if isinstance(value, tuple):  # replace the background color
-                if len(value) == 3:
-                    value += (255,)
-                img_update = remove(img_update, bgcolor=value)
-            if isinstance(value, str):
-                if confirm_rembg_models(value):
-                    img_update = remove(img_update, session=new_session(value))
-                else:
-                    img_update = remove(img_update)
-        elif "bgcolor" in k.lower():
-            if isinstance(value, list):
-                value = tuple(value)
-            if isinstance(value, tuple):  # replace the background color
-                if len(value) == 3:
-                    value += (255,)
-                img_update = remove(img_update, bgcolor=value)
-    if filter_kws:
-        for filter_name, filter_value in filter_kws.items():
-            img_update = apply_filter(img_update, filter_name, filter_value)
-    # Display the image if requested
-    if show:
-        if figsize is None:
-            plt.figure(dpi=dpi)
-        else:
-            plt.figure(figsize=figsize, dpi=dpi)
-        plt.imshow(img_update)
-        plt.axis("on") if show_axis else plt.axis("off")
-    return img_update
-
-
-from sklearn.decomposition import PCA
-from skimage import transform, feature, filters, measure
-from skimage.color import rgb2gray
-from scipy.fftpack import fftshift, fft2
-import numpy as np
-import cv2  # Used for template matching
-
-
-def crop_black_borders(image):
-    """Crop the black borders from a rotated image."""
-    # Convert the image to grayscale if it's not already
-    if image.ndim == 3:
-        gray_image = color.rgb2gray(image)
-    else:
-        gray_image = image
-
-    # Find all the non-black (non-zero) pixels
-    mask = gray_image > 0  # Mask for non-black pixels (assuming black is zero)
-    coords = np.column_stack(np.where(mask))
-
-    # Get the bounding box of non-black pixels
-    if coords.any():  # Check if there are any non-black pixels
-        y_min, x_min = coords.min(axis=0)
-        y_max, x_max = coords.max(axis=0)
-
-        # Crop the image to the bounding box
-        cropped_image = image[y_min : y_max + 1, x_min : x_max + 1]
-    else:
-        # If the image is completely black (which shouldn't happen), return the original image
-        cropped_image = image
-
-    return cropped_image
-
-
-def detect_angle(image, by="median", template=None):
-    """Detect the angle of rotation using various methods."""
-    # Convert to grayscale
-    gray_image = rgb2gray(image)
-
-    # Detect edges using Canny edge detector
-    edges = feature.canny(gray_image, sigma=2)
-
-    # Use Hough transform to detect lines
-    lines = transform.probabilistic_hough_line(edges)
-
-    if not lines and any(["me" in by, "pca" in by]):
-        print("No lines detected. Adjust the edge detection parameters.")
-        return 0
-
-    # Hough Transform-based angle detection (Median/Mean)
-    if "me" in by:
-        angles = []
-        for line in lines:
-            (x0, y0), (x1, y1) = line
-            angle = np.arctan2(y1 - y0, x1 - x0) * 180 / np.pi
-            if 80 < abs(angle) < 100:
-                angles.append(angle)
-        if not angles:
-            return 0
-        if "di" in by:
-            median_angle = np.median(angles)
-            rotation_angle = (
-                90 - median_angle if median_angle > 0 else -90 - median_angle
-            )
-
-            return rotation_angle
-        else:
-            mean_angle = np.mean(angles)
-            rotation_angle = 90 - mean_angle if mean_angle > 0 else -90 - mean_angle
-
-            return rotation_angle
-
-    # PCA-based angle detection
-    elif "pca" in by:
-        y, x = np.nonzero(edges)
-        if len(x) == 0:
-            return 0
-        pca = PCA(n_components=2)
-        pca.fit(np.vstack((x, y)).T)
-        angle = np.arctan2(pca.components_[0, 1], pca.components_[0, 0]) * 180 / np.pi
-        return angle
-
-    # Gradient Orientation-based angle detection
-    elif "gra" in by:
-        gx, gy = np.gradient(gray_image)
-        angles = np.arctan2(gy, gx) * 180 / np.pi
-        hist, bin_edges = np.histogram(angles, bins=360, range=(-180, 180))
-        return bin_edges[np.argmax(hist)]
-
-    # Template Matching-based angle detection
-    elif "temp" in by:
-        if template is None:
-            # Automatically extract a template from the center of the image
-            height, width = gray_image.shape
-            center_x, center_y = width // 2, height // 2
-            size = (
-                min(height, width) // 4
-            )  # Size of the template as a fraction of image size
-            template = gray_image[
-                center_y - size : center_y + size, center_x - size : center_x + size
-            ]
-        best_angle = None
-        best_corr = -1
-        for angle in range(0, 180, 1):  # Checking every degree
-            rotated_template = transform.rotate(template, angle)
-            res = cv2.matchTemplate(gray_image, rotated_template, cv2.TM_CCOEFF)
-            _, max_val, _, _ = cv2.minMaxLoc(res)
-            if max_val > best_corr:
-                best_corr = max_val
-                best_angle = angle
-        return best_angle
-
-    # Image Moments-based angle detection
-    elif "mo" in by:
-        moments = measure.moments_central(gray_image)
-        angle = (
-            0.5
-            * np.arctan2(2 * moments[1, 1], moments[0, 2] - moments[2, 0])
-            * 180
-            / np.pi
-        )
-        return angle
-
-    # Fourier Transform-based angle detection
-    elif "fft" in by:
-        f = fft2(gray_image)
-        fshift = fftshift(f)
-        magnitude_spectrum = np.log(np.abs(fshift) + 1)
-        rows, cols = magnitude_spectrum.shape
-        r, c = np.unravel_index(np.argmax(magnitude_spectrum), (rows, cols))
-        angle = np.arctan2(r - rows // 2, c - cols // 2) * 180 / np.pi
-        return angle
-
-    else:
-        print(f"Unknown method {by}")
-        return 0
-
-
 def imgsets(img, **kwargs):
     """
     Apply various enhancements and filters to an image using PIL's ImageEnhance and ImageFilter modules.
@@ -3179,7 +3191,9 @@ def imgsets(img, **kwargs):
         if "shar" in k.lower():
             enhancer = ImageEnhance.Sharpness(img_update)
             img_update = enhancer.enhance(value)
-        elif "col" in k.lower() and "bg" not in k.lower():
+        elif all(
+            ["col" in k.lower(), "bg" not in k.lower(), "background" not in k.lower()]
+        ):
             enhancer = ImageEnhance.Color(img_update)
             img_update = enhancer.enhance(value)
         elif "contr" in k.lower():
@@ -3201,6 +3215,9 @@ def imgsets(img, **kwargs):
             img_update = img_update.rotate(value)
 
         elif "si" in k.lower():
+            if isinstance(value, tuple):
+                value = list(value)
+            value = [int(i) for i in value]
             img_update = img_update.resize(value)
         elif "thum" in k.lower():
             img_update.thumbnail(value)
@@ -3221,21 +3238,7 @@ def imgsets(img, **kwargs):
                 session = new_session("isnet-general-use")
                 img_update = remove(img_update, session=session)
             elif value and isinstance(value, (int, float, list)):
-                print(
-                    'example usage: {"rm":[alpha_matting_background_threshold(20),alpha_matting_foreground_threshold(270),alpha_matting_erode_sive(11)]}'
-                )
                 print("https://github.com/danielgatis/rembg/blob/main/USAGE.md")
-                #     ###            Parameters:
-                #         data (Union[bytes, PILImage, np.ndarray]): The input image data.
-                #         alpha_matting (bool, optional): Flag indicating whether to use alpha matting. Defaults to False.
-                #         alpha_matting_foreground_threshold (int, optional): Foreground threshold for alpha matting. Defaults to 240.
-                #         alpha_matting_background_threshold (int, optional): Background threshold for alpha matting. Defaults to 10.
-                #         alpha_matting_erode_size (int, optional): Erosion size for alpha matting. Defaults to 10.
-                #         session (Optional[BaseSession], optional): A session object for the 'u2net' model. Defaults to None.
-                #         only_mask (bool, optional): Flag indicating whether to return only the binary masks. Defaults to False.
-                #         post_process_mask (bool, optional): Flag indicating whether to post-process the masks. Defaults to False.
-                #         bgcolor (Optional[Tuple[int, int, int, int]], optional): Background color for the cutout image. Defaults to None.
-                #  ###
                 if isinstance(value, int):
                     value = [value]
                 if len(value) < 2:
@@ -3294,16 +3297,6 @@ def imgsets(img, **kwargs):
     return img_update
 
 
-# # usage:
-# img = imgsets(
-#     fpath,
-#     sets={"rota": -5},
-#     dpi=200,
-#     filter_kws={"EMBOSS": 5, "sharpen": 5, "EDGE_ENHANCE_MORE": 10},
-#     show_axis=True,
-# )
-
-
 def thumbnail(dir_img_list, figsize=(10, 10), dpi=100, dir_save=None, kind=".png"):
     """
     Display a thumbnail figure of all images in the specified directory.
@@ -4248,7 +4241,7 @@ format_excel(
     print(f"Formatted Excel file saved as:\n{filename}")
 
 
-from IPython.display import display, HTML, Markdown, Image
+from IPython.display import display, HTML, Markdown
 
 
 def preview(var):
@@ -4528,3 +4521,709 @@ def df_sort_values(df, column, by=None, ascending=True, inplace=False, **kwargs)
 # display(sorted_df_month)
 # df_sort_values(df_month, "month", month_order, ascending=True, inplace=True)
 # display(df_month)
+
+
+def df_cluster(
+    data: pd.DataFrame,
+    columns: Optional[list] = None,
+    n_clusters: Optional[int] = None,
+    range_n_clusters: Union[range, np.ndarray] = range(2, 11),
+    scale: bool = True,
+    plot: Union[str, list] = "all",
+    inplace: bool = True,
+    ax: Optional[plt.Axes] = None,
+) -> tuple[pd.DataFrame, int, Optional[plt.Axes]]:
+    from sklearn.preprocessing import StandardScaler
+    from sklearn.cluster import KMeans
+    from sklearn.metrics import silhouette_score, silhouette_samples
+    import seaborn as sns
+    import numpy as np
+    import pandas as pd
+    import matplotlib.pyplot as plt
+    import seaborn as sns
+
+    """
+    Performs clustering analysis on the provided feature matrix using K-Means.
+
+    Parameters:
+        X (np.ndarray):
+            A 2D numpy array or DataFrame containing numerical feature data,
+            where each row corresponds to an observation and each column to a feature.
+
+        range_n_clusters (range):
+            A range object specifying the number of clusters to evaluate for K-Means clustering.
+            Default is range(2, 11), meaning it will evaluate from 2 to 10 clusters.
+
+        scale (bool):
+            A flag indicating whether to standardize the features before clustering.
+            Default is True, which scales the data to have a mean of 0 and variance of 1.
+
+        plot (bool):
+            A flag indicating whether to generate visualizations of the clustering analysis.
+            Default is True, which will plot silhouette scores, inertia, and other relevant plots.
+    Returns:
+        tuple: 
+            A tuple containing the modified DataFrame with cluster labels, 
+            the optimal number of clusters, and the Axes object (if any).
+    """
+    X = data[columns].values if columns is not None else data.values
+
+    silhouette_avg_scores = []
+    inertia_scores = []
+
+    # Standardize the features
+    if scale:
+        scaler = StandardScaler()
+        X = scaler.fit_transform(X)
+
+    for n_cluster in range_n_clusters:
+        kmeans = KMeans(n_clusters=n_cluster, random_state=42)
+        cluster_labels = kmeans.fit_predict(X)
+
+        silhouette_avg = silhouette_score(X, cluster_labels)
+        silhouette_avg_scores.append(silhouette_avg)
+        inertia_scores.append(kmeans.inertia_)
+        print(
+            f"For n_clusters = {n_cluster}, the average silhouette_score is : {silhouette_avg:.4f}"
+        )
+
+    # Determine the optimal number of clusters based on the maximum silhouette score
+    if n_clusters is None:
+        n_clusters = range_n_clusters[np.argmax(silhouette_avg_scores)]
+    print(f"n_clusters = {n_clusters}")
+
+    # Apply K-Means Clustering with Optimal Number of Clusters
+    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
+    cluster_labels = kmeans.fit_predict(X)
+
+    if plot:
+        # ! Interpreting the plots from your K-Means clustering analysis
+        # ! 1. Silhouette Score and Inertia vs Number of Clusters
+        # Description:
+        # This plot has two y-axes: the left y-axis represents the Silhouette Score, and the right y-axis
+        # represents Inertia.
+        # The x-axis represents the number of clusters (k).
+
+        # Interpretation:
+
+        # Silhouette Score:
+        # Ranges from -1 to 1, where a score close to 1 indicates that points are well-clustered, while a
+        # score close to -1 indicates that points might be incorrectly clustered.
+        # A higher silhouette score generally suggests that the data points are appropriately clustered.
+        # Look for the highest value to determine the optimal number of clusters.
+
+        # Inertia:
+        # Represents the sum of squared distances from each point to its assigned cluster center.
+        # Lower inertia values indicate tighter clusters.
+        # As the number of clusters increases, inertia typically decreases, but the rate of decrease
+        # may slow down, indicating diminishing returns for additional clusters.
+
+        # Optimal Number of Clusters:
+        # You can identify an optimal number of clusters where the silhouette score is maximized and
+        # inertia starts to plateau (the "elbow" point).
+        # This typically suggests that increasing the number of clusters further yields less meaningful
+        # separations.
+        if ax is None:
+            _, ax = plt.subplots(figsize=inch2cm(10, 6))
+        color = "tab:blue"
+        ax.plot(
+            range_n_clusters,
+            silhouette_avg_scores,
+            marker="o",
+            color=color,
+            label="Silhouette Score",
+        )
+        ax.set_xlabel("Number of Clusters")
+        ax.set_ylabel("Silhouette Score", color=color)
+        ax.tick_params(axis="y", labelcolor=color)
+        # add right axis: inertia
+        ax2 = ax.twinx()
+        color = "tab:red"
+        ax2.set_ylabel("Inertia", color=color)
+        ax2.plot(
+            range_n_clusters,
+            inertia_scores,
+            marker="x",
+            color=color,
+            label="Inertia",
+        )
+        ax2.tick_params(axis="y", labelcolor=color)
+
+        plt.title("Silhouette Score and Inertia vs Number of Clusters")
+        plt.xticks(range_n_clusters)
+        plt.grid()
+        plt.axvline(x=n_clusters, linestyle="--", color="r", label="Optimal n_clusters")
+        # ! 2. Elbow Method Plot
+        # Description:
+        # This plot shows the Inertia against the number of clusters.
+
+        # Interpretation:
+        # The elbow point is where the inertia begins to decrease at a slower rate. This point suggests that
+        # adding more clusters beyond this point does not significantly improve the clustering performance.
+        # Look for a noticeable bend in the curve to identify the optimal number of clusters, indicated by the
+        # vertical dashed line.
+        # Inertia plot
+        plt.figure(figsize=inch2cm(10, 6))
+        plt.plot(range_n_clusters, inertia_scores, marker="o")
+        plt.title("Elbow Method for Optimal k")
+        plt.xlabel("Number of clusters")
+        plt.ylabel("Inertia")
+        plt.grid()
+        plt.axvline(
+            x=np.argmax(silhouette_avg_scores) + 2,
+            linestyle="--",
+            color="r",
+            label="Optimal n_clusters",
+        )
+        plt.legend()
+        # ! Silhouette Plots
+        # 3. Silhouette Plot for Various Clusters
+        # Description:
+        # This horizontal bar plot shows the silhouette coefficient values for each sample, organized by cluster.
+
+        # Interpretation:
+        # Each bar represents the silhouette score of a sample within a specific cluster. Longer bars indicate
+        # that the samples are well-clustered.
+        # The height of the bars shows how similar points within the same cluster are to one another compared to
+        # points in other clusters.
+        # The vertical red dashed line indicates the average silhouette score for all samples.
+        # You want the majority of silhouette values to be above the average line, indicating that most points
+        # are well-clustered.
+
+        # 以下代码不用再跑一次了
+        # n_clusters = (
+        #     np.argmax(silhouette_avg_scores) + 2
+        # )  # Optimal clusters based on max silhouette score
+        # kmeans = KMeans(n_clusters=n_clusters, random_state=42)
+        # cluster_labels = kmeans.fit_predict(X)
+        silhouette_vals = silhouette_samples(X, cluster_labels)
+
+        plt.figure(figsize=inch2cm(10, 6))
+        y_lower = 10
+        for i in range(n_clusters):
+            # Aggregate the silhouette scores for samples belonging to cluster i
+            ith_cluster_silhouette_values = silhouette_vals[cluster_labels == i]
+
+            # Sort the values
+            ith_cluster_silhouette_values.sort()
+
+            size_cluster_i = ith_cluster_silhouette_values.shape[0]
+            y_upper = y_lower + size_cluster_i
+
+            # Create a horizontal bar plot for the silhouette scores
+            plt.barh(range(y_lower, y_upper), ith_cluster_silhouette_values, height=0.5)
+
+            # Label the silhouette scores
+            plt.text(-0.05, (y_lower + y_upper) / 2, str(i + 2))
+            y_lower = y_upper + 10  # 10 for the 0 samples
+
+        plt.title("Silhouette Plot for the Various Clusters")
+        plt.xlabel("Silhouette Coefficient Values")
+        plt.ylabel("Cluster Label")
+        plt.axvline(x=np.mean(silhouette_vals), color="red", linestyle="--")
+
+        df_clusters = pd.DataFrame(
+            X, columns=[f"Feature {i+1}" for i in range(X.shape[1])]
+        )
+        df_clusters["Cluster"] = cluster_labels
+        # ! pairplot of the clusters
+        # Overview of the Pairplot
+        # Axes and Grid:
+        # The pairplot creates a grid of scatter plots for each pair of features in your dataset.
+        # Each point in the scatter plots represents a sample from your dataset, colored according to its cluster assignment.
+
+        # Diagonal Elements:
+        # The diagonal plots usually show the distribution of each feature. In this case, since X.shape[1] <= 4,
+        # there will be a maximum of four features plotted against each other. The diagonal could display histograms or
+        # kernel density estimates (KDE) for each feature.
+
+        # Interpretation of the Pairplot
+
+        # Feature Relationships:
+        # Look at each scatter plot in the off-diagonal plots. Each plot shows the relationship between two features. Points that
+        # are close together in the scatter plot suggest similar values for those features.
+        # Cluster Separation: You want to see clusters of different colors (representing different clusters) that are visually distinct.
+        # Good separation indicates that the clustering algorithm effectively identified different groups within your data.
+        # Overlapping Points: If points from different clusters overlap significantly in any scatter plot, it indicates that those clusters
+        # might not be distinct in terms of the two features being compared.
+        # Cluster Characteristics:
+        # Shape and Distribution: Observe the shape of the clusters. Are they spherical, elongated, or irregular? This can give insights
+        # into how well the K-Means (or other clustering methods) has performed:
+        # Spherical Clusters: Indicates that clusters are well defined and separated.
+        # Elongated Clusters: May suggest that the algorithm is capturing variations along specific axes but could benefit from adjustments
+        # in clustering parameters or methods.
+        # Feature Influence: Identify which features contribute most to cluster separation. For instance, if you see that one feature
+        # consistently separates two clusters, it may be a key factor for clustering.
+        # Diagonal Histograms/KDE:
+        # The diagonal plots show the distribution of individual features across all samples. Look for:
+        # Distribution Shape: Is the distribution unimodal, bimodal, skewed, or uniform?
+        # Concentration: Areas with a high density of points may indicate that certain values are more common among samples.
+        # Differences Among Clusters: If you see distinct peaks in the histograms for different clusters, it suggests that those clusters are
+        # characterized by specific ranges of feature values.
+        # Example Observations
+        # Feature 1 vs. Feature 2: If there are clear, well-separated clusters in this scatter plot, it suggests that these two features
+        # effectively distinguish between the clusters.
+        # Feature 3 vs. Feature 4: If you observe significant overlap between clusters in this plot, it may indicate that these features do not
+        # provide a strong basis for clustering.
+        # Diagonal Plots: If you notice that one cluster has a higher density of points at lower values for a specific feature, while another
+        # cluster is concentrated at higher values, this suggests that this feature is critical for differentiating those clusters.
+
+        # Pairplot of the clusters
+        # * 为什么要限制到4个features?
+        # 2 features=1 scatter plot            # 3 features=3 scatter plots
+        # 4 features=6 scatter plots           # 5 features=10 scatter plots
+        # 6 features=15 scatter plots          # 10 features=45 scatter plots
+        # Pairplot works well with low-dimensional data, 如果维度比较高的话, 子图也很多,失去了它的意义
+        if X.shape[1] <= 6:
+            plt.figure(figsize=(8, 4))
+            sns.pairplot(df_clusters, hue="Cluster", palette="tab10")
+            plt.suptitle("Pairplot of Clusters", y=1.02)
+
+    # Add cluster labels to the DataFrame or modify in-place
+    if inplace:  # replace the oringinal data
+        data["Cluster"] = cluster_labels
+        return None, n_clusters, kmeans, ax  # Return None when inplace is True
+    else:
+        data_copy = data.copy()
+        data_copy["Cluster"] = cluster_labels
+        return data_copy, n_clusters, kmeans, ax
+
+
+# example:
+# clustering_features = [marker + "_log" for marker in markers]
+# df_cluster(data, columns=clustering_features, n_clusters=3,range_n_clusters=np.arange(3, 7))
+
+"""
+# You're on the right track, but let's clarify how PCA and clustering (like KMeans) work, especially 
+# in the context of your dataset with 7 columns and 23,121 rows.
+
+# Principal Component Analysis (PCA)
+# Purpose of PCA:
+# PCA is a dimensionality reduction technique. It transforms your dataset from a high-dimensional space 
+# (in your case, 7 dimensions corresponding to your 7 columns) to a lower-dimensional space while 
+# retaining as much variance (information) as possible.
+# How PCA Works:
+# PCA computes new features called "principal components" that are linear combinations of the original 
+# features.
+# The first principal component captures the most variance, the second captures the next most variance 
+# (orthogonal to the first), and so on.
+# If you set n_components=2, for example, PCA will reduce your dataset from 7 columns to 2 columns. 
+# This helps in visualizing and analyzing the data with fewer dimensions.
+# Result of PCA:
+# After applying PCA, your original dataset with 7 columns will be transformed into a new dataset with 
+# the specified number of components (e.g., 2 or 3).
+# The transformed dataset will have fewer columns but should capture most of the important information 
+# from the original dataset.
+
+# Clustering (KMeans)
+# Purpose of Clustering:
+# Clustering is used to group data points based on their similarities. KMeans, specifically, partitions 
+# your data into a specified number of clusters (groups).
+# How KMeans Works:
+# KMeans assigns each data point to one of the k clusters based on the feature space (original or 
+# PCA-transformed).
+# It aims to minimize the variance within each cluster while maximizing the variance between clusters.
+# It does not classify the data in each column independently; instead, it considers the overall similarity 
+# between data points based on their features.
+# Result of KMeans:
+# The output will be cluster labels for each data point (e.g., which cluster a particular observation 
+# belongs to).
+# You can visualize how many groups were formed and analyze the characteristics of each cluster.
+
+# Summary
+# PCA reduces the number of features (columns) in your dataset, transforming it into a lower-dimensional
+# space.
+# KMeans then classifies data points based on the features of the transformed dataset (or the original 
+# if you choose) into different subgroups (clusters).
+# By combining these techniques, you can simplify the complexity of your data and uncover patterns that 
+# might not be visible in the original high-dimensional space. Let me know if you have further questions!
+"""
+
+
+def df_reducer(
+    data: pd.DataFrame,
+    columns: Optional[List[str]] = None,
+    method: str = "umap",  # 'pca', 'umap'
+    n_components: int = 2,  # Default for umap, but 50 for PCA
+    umap_neighbors: int = 15,  # Default
+    umap_min_dist: float = 0.1,  # Default
+    scale: bool = True,
+    fill_missing: bool = True,
+    debug: bool = False,
+    inplace: bool = True,  # replace the oringinal data
+) -> pd.DataFrame:
+    """
+    Reduces the dimensionality of the selected DataFrame using PCA or UMAP.
+
+    Parameters:
+    -----------
+    data : pd.DataFrame
+        The input DataFrame (samples x features).
+
+    columns : List[str], optional
+        List of column names to reduce. If None, all columns are used.
+
+    method : str, optional, default="umap"
+        Dimensionality reduction method, either "pca" or "umap".
+
+    n_components : int, optional, default=50
+        Number of components for PCA or UMAP.
+
+    umap_neighbors : int, optional, default=15
+        Number of neighbors considered for UMAP embedding.
+
+    umap_min_dist : float, optional, default=0.1
+        Minimum distance between points in UMAP embedding.
+
+    scale : bool, optional, default=True
+        Whether to scale the data using StandardScaler.
+
+    fill_missing : bool, optional, default=True
+        Whether to fill missing values using the mean before applying PCA/UMAP.
+
+    Returns:
+    --------
+    reduced_df : pd.DataFrame
+        DataFrame with the reduced dimensions.
+    """
+    from sklearn.decomposition import PCA
+    from sklearn.preprocessing import StandardScaler
+    import umap
+    from sklearn.impute import SimpleImputer
+
+    # Select columns if specified, else use all columns
+    X = data[columns].values if columns else data.values
+
+    # Handle missing values
+    if fill_missing:
+        imputer = SimpleImputer(strategy="mean")
+        X = imputer.fit_transform(X)
+
+    # Optionally scale the data
+    if scale:
+        scaler = StandardScaler()
+        X = scaler.fit_transform(X)
+
+    # Check valid method input
+    if method not in ["pca", "umap"]:
+        raise ValueError(f"Invalid method '{method}'. Choose 'pca' or 'umap'.")
+
+    # Apply PCA if selected
+    if method == "pca":
+        if n_components is None:
+            # to get the n_components with threshold method:
+            pca = PCA()
+            pca_result = pca.fit_transform(X)
+
+            # Calculate explained variance
+            explained_variance = pca.explained_variance_ratio_
+            # Cumulative explained variance
+            cumulative_variance = np.cumsum(explained_variance)
+            # Set a threshold for cumulative variance
+            threshold = 0.95  # Example threshold
+            n_components = (
+                np.argmax(cumulative_variance >= threshold) + 1
+            )  # Number of components to retain
+            if debug:
+                # debug:
+                # Plot the cumulative explained variance
+                plt.figure(figsize=(8, 5))
+                plt.plot(
+                    range(1, len(cumulative_variance) + 1),
+                    cumulative_variance,
+                    marker="o",
+                    linestyle="-",
+                )
+                plt.title("Cumulative Explained Variance by Principal Components")
+                plt.xlabel("Number of Principal Components")
+                plt.ylabel("Cumulative Explained Variance")
+                plt.xticks(range(1, len(cumulative_variance) + 1))
+                # Add horizontal line for the threshold
+                plt.axhline(
+                    y=threshold, color="r", linestyle="--", label="Threshold (95%)"
+                )
+                # Add vertical line for n_components
+                plt.axvline(
+                    x=n_components,
+                    color="g",
+                    linestyle="--",
+                    label=f"n_components = {n_components}",
+                )
+                plt.legend()
+                plt.grid()
+        pca = PCA(n_components=n_components)
+        X_reduced = pca.fit_transform(X)
+        print(f"PCA completed: Reduced to {n_components} components.")
+
+    # Apply UMAP if selected
+    elif method == "umap":
+        umap_reducer = umap.UMAP(
+            n_neighbors=umap_neighbors,
+            min_dist=umap_min_dist,
+            n_components=n_components,
+        )
+        X_reduced = umap_reducer.fit_transform(X)
+        print(f"UMAP completed: Reduced to {n_components} components.")
+
+    # Return reduced data as a new DataFrame with the same index
+    reduced_df = pd.DataFrame(X_reduced, index=data.index)
+
+    if inplace:
+        # Replace or add new columns based on n_components
+        for col_idx in range(n_components):
+            data[f"Component_{col_idx+1}"] = reduced_df.iloc[:, col_idx]
+        return None  # No return when inplace=True
+
+    return reduced_df
+
+
+# example:
+# df_reducer(data=data_log, columns=markers, n_components=2)
+
+
+def plot_cluster(
+    data: pd.DataFrame,
+    labels: np.ndarray,
+    metrics: dict = None,
+    cmap="tab20",
+    true_labels: Optional[np.ndarray] = None,
+) -> None:
+    """
+    Visualize clustering results with various plots.
+
+    Parameters:
+    -----------
+    data : pd.DataFrame
+        The input data used for clustering.
+    labels : np.ndarray
+        Cluster labels assigned to each point.
+    metrics : dict
+        Dictionary containing evaluation metrics from evaluate_cluster function.
+    true_labels : Optional[np.ndarray], default=None
+        Ground truth labels, if available.
+    """
+    import seaborn as sns
+    from sklearn.metrics import silhouette_samples
+
+    if metrics is None:
+        metrics = evaluate_cluster(data=data, labels=labels, true_labels=true_labels)
+
+    # 1. Scatter Plot of Clusters
+    plt.figure(figsize=(15, 6))
+    plt.subplot(1, 3, 1)
+    plt.scatter(data.iloc[:, 0], data.iloc[:, 1], c=labels, cmap=cmap, s=20)
+    plt.title("Cluster Scatter Plot")
+    plt.xlabel("Component 1")
+    plt.ylabel("Component 2")
+    plt.colorbar(label="Cluster Label")
+    plt.grid()
+
+    # 2. Silhouette Plot
+    if "Silhouette Score" in metrics:
+        silhouette_vals = silhouette_samples(data, labels)
+        plt.subplot(1, 3, 2)
+        y_lower = 10
+        for i in range(len(set(labels))):
+            # Aggregate the silhouette scores for samples belonging to the current cluster
+            cluster_silhouette_vals = silhouette_vals[labels == i]
+            cluster_silhouette_vals.sort()
+            size_cluster_i = cluster_silhouette_vals.shape[0]
+            y_upper = y_lower + size_cluster_i
+
+            plt.fill_betweenx(np.arange(y_lower, y_upper), 0, cluster_silhouette_vals)
+            plt.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
+            y_lower = y_upper + 10  # 10 for the 0 samples
+
+        plt.title("Silhouette Plot")
+        plt.xlabel("Silhouette Coefficient Values")
+        plt.ylabel("Cluster Label")
+        plt.axvline(x=metrics["Silhouette Score"], color="red", linestyle="--")
+        plt.grid()
+
+    # 3. Metrics Plot
+    plt.subplot(1, 3, 3)
+    metric_names = ["Davies-Bouldin Index", "Calinski-Harabasz Index"]
+    metric_values = [
+        metrics["Davies-Bouldin Index"],
+        metrics["Calinski-Harabasz Index"],
+    ]
+
+    if true_labels is not None:
+        metric_names += ["Homogeneity Score", "Completeness Score", "V-Measure"]
+        metric_values += [
+            metrics["Homogeneity Score"],
+            metrics["Completeness Score"],
+            metrics["V-Measure"],
+        ]
+
+    plt.barh(metric_names, metric_values, color="lightblue")
+    plt.title("Clustering Metrics")
+    plt.xlabel("Score")
+    plt.axvline(x=0, color="gray", linestyle="--")
+    plt.grid()
+    plt.tight_layout()
+
+
+def evaluate_cluster(
+    data: pd.DataFrame, labels: np.ndarray, true_labels: Optional[np.ndarray] = None
+) -> dict:
+    """
+    Evaluate clustering performance using various metrics.
+
+    Parameters:
+    -----------
+    data : pd.DataFrame
+        The input data used for clustering.
+    labels : np.ndarray
+        Cluster labels assigned to each point.
+    true_labels : Optional[np.ndarray], default=None
+        Ground truth labels, if available.
+
+    Returns:
+    --------
+    metrics : dict
+        Dictionary containing evaluation metrics.
+
+    1. Silhouette Score:
+    The silhouette score measures how similar an object is to its own cluster (cohesion) compared to
+    how similar it is to other clusters (separation). The score ranges from -1 to +1:
+    +1: Indicates that the data point is very far from the neighboring clusters and well clustered.
+    0: Indicates that the data point is on or very close to the decision boundary between two neighboring
+    clusters.
+    -1: Indicates that the data point might have been assigned to the wrong cluster.
+
+    Interpretation:
+    A higher average silhouette score indicates better-defined clusters.
+    If the score is consistently high (above 0.5), it suggests that the clusters are well separated.
+    A score near 0 may indicate overlapping clusters, while negative scores suggest points may have
+    been misclassified.
+
+    2. Davies-Bouldin Index:
+    The Davies-Bouldin Index (DBI) measures the average similarity ratio of each cluster with its
+    most similar cluster. The index values range from 0 to ∞, with lower values indicating better clustering.
+    It is defined as the ratio of within-cluster distances to between-cluster distances.
+
+    Interpretation:
+    A lower DBI value indicates that the clusters are compact and well-separated.
+    Ideally, you want to minimize the Davies-Bouldin Index. If your DBI value is above 1, this indicates
+    that your clusters might not be well-separated.
+
+    3. Adjusted Rand Index (ARI):
+    The Adjusted Rand Index (ARI) is a measure of the similarity between two data clusterings. The ARI
+    score ranges from -1 to +1:
+    1: Indicates perfect agreement between the two clusterings.
+    0: Indicates that the clusterings are no better than random.
+    Negative values: Indicate less agreement than expected by chance.
+
+    Interpretation:
+    A higher ARI score indicates better clustering, particularly if it's close to 1.
+    An ARI score of 0 or lower suggests that the clustering results do not represent the true labels
+    well, indicating a poor clustering performance.
+
+    4. Calinski-Harabasz Index:
+    The Calinski-Harabasz Index (also known as the Variance Ratio Criterion) evaluates the ratio of
+    the sum of between-cluster dispersion to within-cluster dispersion. Higher values indicate better clustering.
+
+    Interpretation:
+    A higher Calinski-Harabasz Index suggests that clusters are dense and well-separated. It is typically
+    used to validate the number of clusters, with higher values favoring more distinct clusters.
+
+    5. Homogeneity Score:
+    The homogeneity score measures how much a cluster contains only members of a single class (if true labels are provided).
+    A score of 1 indicates perfect homogeneity, where all points in a cluster belong to the same class.
+
+    Interpretation:
+    A higher homogeneity score indicates that the clustering result is pure, meaning the clusters are composed
+    of similar members. Lower values indicate mixed clusters, suggesting poor clustering performance.
+
+    6. Completeness Score:
+    The completeness score evaluates how well all members of a given class are assigned to the same cluster.
+    A score of 1 indicates perfect completeness, meaning all points in a true class are assigned to a single cluster.
+
+    Interpretation:
+    A higher completeness score indicates that the clustering effectively groups all instances of a class together.
+    Lower values suggest that some instances of a class are dispersed among multiple clusters.
+
+    7. V-Measure:
+    The V-measure is the harmonic mean of homogeneity and completeness, giving a balanced measure of clustering performance.
+
+    Interpretation:
+    A higher V-measure score indicates that the clusters are both homogenous (pure) and complete (cover all members of a class).
+    Scores closer to 1 indicate better clustering quality.
+    """
+    from sklearn.metrics import (
+        silhouette_score,
+        davies_bouldin_score,
+        adjusted_rand_score,
+        calinski_harabasz_score,
+        homogeneity_score,
+        completeness_score,
+        v_measure_score,
+    )
+
+    metrics = {}
+    unique_labels = set(labels)
+    if len(unique_labels) > 1 and len(unique_labels) < len(data):
+        # Calculate Silhouette Score
+        try:
+            metrics["Silhouette Score"] = silhouette_score(data, labels)
+        except Exception as e:
+            metrics["Silhouette Score"] = np.nan
+            print(f"Silhouette Score calculation failed: {e}")
+
+        # Calculate Davies-Bouldin Index
+        try:
+            metrics["Davies-Bouldin Index"] = davies_bouldin_score(data, labels)
+        except Exception as e:
+            metrics["Davies-Bouldin Index"] = np.nan
+            print(f"Davies-Bouldin Index calculation failed: {e}")
+
+        # Calculate Calinski-Harabasz Index
+        try:
+            metrics["Calinski-Harabasz Index"] = calinski_harabasz_score(data, labels)
+        except Exception as e:
+            metrics["Calinski-Harabasz Index"] = np.nan
+            print(f"Calinski-Harabasz Index calculation failed: {e}")
+
+        # Calculate Adjusted Rand Index if true labels are provided
+        if true_labels is not None:
+            try:
+                metrics["Adjusted Rand Index"] = adjusted_rand_score(
+                    true_labels, labels
+                )
+            except Exception as e:
+                metrics["Adjusted Rand Index"] = np.nan
+                print(f"Adjusted Rand Index calculation failed: {e}")
+
+            # Calculate Homogeneity Score
+            try:
+                metrics["Homogeneity Score"] = homogeneity_score(true_labels, labels)
+            except Exception as e:
+                metrics["Homogeneity Score"] = np.nan
+                print(f"Homogeneity Score calculation failed: {e}")
+
+            # Calculate Completeness Score
+            try:
+                metrics["Completeness Score"] = completeness_score(true_labels, labels)
+            except Exception as e:
+                metrics["Completeness Score"] = np.nan
+                print(f"Completeness Score calculation failed: {e}")
+
+            # Calculate V-Measure
+            try:
+                metrics["V-Measure"] = v_measure_score(true_labels, labels)
+            except Exception as e:
+                metrics["V-Measure"] = np.nan
+                print(f"V-Measure calculation failed: {e}")
+    else:
+        # Metrics cannot be computed with 1 cluster or all points as noise
+        metrics["Silhouette Score"] = np.nan
+        metrics["Davies-Bouldin Index"] = np.nan
+        metrics["Calinski-Harabasz Index"] = np.nan
+        if true_labels is not None:
+            metrics["Adjusted Rand Index"] = np.nan
+            metrics["Homogeneity Score"] = np.nan
+            metrics["Completeness Score"] = np.nan
+            metrics["V-Measure"] = np.nan
+
+    return metrics