py2ls 0.2.4.25__py3-none-any.whl → 0.2.4.27__py3-none-any.whl

py2ls/ml2ls.py

@@ -31,6 +31,7 @@ from sklearn.metrics import (
     average_precision_score,
 )
 from typing import Dict, Any, Optional, List, Union
+import os, json
 import numpy as np
 import pandas as pd
 from . import ips
@@ -49,7 +50,13 @@ logger = logging.getLogger()
 warnings.filterwarnings("ignore", category=UserWarning)
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.neighbors import KNeighborsClassifier
-
+#* set random_state global 
+import torch 
+import random
+random_state=1
+random.seed(random_state)
+np.random.seed(random_state)
+torch.manual_seed(random_state)
 
 def features_knn(
     x_train: pd.DataFrame, y_train: pd.Series, knn_params: dict
@@ -594,7 +601,7 @@ def get_features(
     """
     from sklearn.compose import ColumnTransformer
     from sklearn.preprocessing import StandardScaler, OneHotEncoder
-
+    from sklearn.model_selection import train_test_split
     # Ensure X and y are DataFrames/Series for consistency
     if isinstance(X, np.ndarray):
         X = pd.DataFrame(X)
@@ -922,10 +929,26 @@ def get_features(
             "feature_importances": feature_importances,
         }
         if all([plot_, dir_save]):
+            
             from datetime import datetime
-
             now_ = datetime.now().strftime("%y%m%d_%H%M%S")
             ips.figsave(dir_save + f"features{now_}.pdf")
+            
+            lists = []
+            for tp in ips.flatten(features_df["type"]):
+                lists.append(
+                    features_df
+                    .loc[features_df["type"] == tp, "feature"]
+                    .tolist()
+                )
+            labels = ips.flatten(features_df["type"])
+            # current_fig = plt.gcf() 
+            # # ax = current_fig.add_subplot(3, 2, 6)
+            # gs = current_fig.add_gridspec(3, 2)
+            # ax = current_fig.add_subplot(gs[:, :])
+            plt.figure(figsize=[6,6])
+            plot.venn(lists, labels, cmap="coolwarm") 
+            ips.figsave(dir_save + f"features{now_}shared_features.pdf")
     else:
         results = {
             "selected_features": pd.DataFrame(),
@@ -1247,22 +1270,25 @@ def plot_validate_features(res_val, is_binary=True, figsize=None):
         nexttile = plot.subplot(figsize=figsize)
         ax = nexttile(subplot_layout[0], subplot_layout[1])
         for i, model_name in enumerate(ips.flatten(res_val["pr_curve"].index)):
-            fpr = res_val["roc_curve"][model_name]["fpr"]
-            tpr = res_val["roc_curve"][model_name]["tpr"]
-            (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"]
-            mean_auc = res_val["roc_curve"][model_name]["auc"]
-            plot_roc_curve(
-                fpr,
-                tpr,
-                mean_auc,
-                lower_ci,
-                upper_ci,
-                model_name=model_name,
-                lw=1.5,
-                color=colors[i],
-                alpha=alpha,
-                ax=ax,
-            )
+            try:
+                fpr = res_val["roc_curve"][model_name]["fpr"]
+                tpr = res_val["roc_curve"][model_name]["tpr"]
+                (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"]
+                mean_auc = res_val["roc_curve"][model_name]["auc"]
+                plot_roc_curve(
+                    fpr,
+                    tpr,
+                    mean_auc,
+                    lower_ci,
+                    upper_ci,
+                    model_name=model_name,
+                    lw=1.5,
+                    color=colors[i],
+                    alpha=alpha,
+                    ax=ax,
+                )
+            except Exception as e:
+                print(e)
         plot.figsets(
             sp=2,
             legend=dict(
@@ -1277,16 +1303,19 @@ def plot_validate_features(res_val, is_binary=True, figsize=None):
 
         ax = nexttile(subplot_layout[0], subplot_layout[1])
         for i, model_name in enumerate(ips.flatten(res_val["pr_curve"].index)):
-            plot_pr_curve(
-                recall=res_val["pr_curve"][model_name]["recall"],
-                precision=res_val["pr_curve"][model_name]["precision"],
-                avg_precision=res_val["pr_curve"][model_name]["avg_precision"],
-                model_name=model_name,
-                color=colors[i],
-                lw=1.5,
-                alpha=alpha,
-                ax=ax,
-            )
+            try:
+                plot_pr_curve(
+                    recall=res_val["pr_curve"][model_name]["recall"],
+                    precision=res_val["pr_curve"][model_name]["precision"],
+                    avg_precision=res_val["pr_curve"][model_name]["avg_precision"],
+                    model_name=model_name,
+                    color=colors[i],
+                    lw=1.5,
+                    alpha=alpha,
+                    ax=ax,
+                )
+            except Exception as e:
+                print(e)
         plot.figsets(
             sp=2,
             legend=dict(
@@ -1314,22 +1343,25 @@ def plot_validate_features(res_val, is_binary=True, figsize=None):
         for iclass, class_ in enumerate(classes):
             ax = nexttile(subplot_layout[0], subplot_layout[1])
             for i, model_name in enumerate(ips.flatten(res_val["pr_curve"].index)):
-                fpr = res_val["roc_curve"][model_name]["fpr"][class_]
-                tpr = res_val["roc_curve"][model_name]["tpr"][class_]
-                (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"][iclass]
-                mean_auc = res_val["roc_curve"][model_name]["auc"][class_]
-                plot_roc_curve(
-                    fpr,
-                    tpr,
-                    mean_auc,
-                    lower_ci,
-                    upper_ci,
-                    model_name=model_name,
-                    lw=1.5,
-                    color=colors[i],
-                    alpha=alpha,
-                    ax=ax,
-                )
+                try:
+                    fpr = res_val["roc_curve"][model_name]["fpr"][class_]
+                    tpr = res_val["roc_curve"][model_name]["tpr"][class_]
+                    (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"][iclass]
+                    mean_auc = res_val["roc_curve"][model_name]["auc"][class_]
+                    plot_roc_curve(
+                        fpr,
+                        tpr,
+                        mean_auc,
+                        lower_ci,
+                        upper_ci,
+                        model_name=model_name,
+                        lw=1.5,
+                        color=colors[i],
+                        alpha=alpha,
+                        ax=ax,
+                    )
+                except Exception as e:
+                    print(e)
             plot.figsets(
                 sp=2,
                 title=class_,
@@ -1345,18 +1377,21 @@ def plot_validate_features(res_val, is_binary=True, figsize=None):
 
             ax = nexttile(subplot_layout[0], subplot_layout[1])
             for i, model_name in enumerate(ips.flatten(res_val["pr_curve"].index)):
-                plot_pr_curve(
-                    recall=res_val["pr_curve"][model_name]["recall"][iclass],
-                    precision=res_val["pr_curve"][model_name]["precision"][iclass],
-                    avg_precision=res_val["pr_curve"][model_name]["avg_precision"][
-                        iclass
-                    ],
-                    model_name=model_name,
-                    color=colors[i],
-                    lw=1.5,
-                    alpha=alpha,
-                    ax=ax,
-                )
+                try:
+                    plot_pr_curve(
+                        recall=res_val["pr_curve"][model_name]["recall"][iclass],
+                        precision=res_val["pr_curve"][model_name]["precision"][iclass],
+                        avg_precision=res_val["pr_curve"][model_name]["avg_precision"][
+                            iclass
+                        ],
+                        model_name=model_name,
+                        color=colors[i],
+                        lw=1.5,
+                        alpha=alpha,
+                        ax=ax,
+                    )
+                except Exception as e:
+                    print(e)
             plot.figsets(
                 sp=2,
                 title=class_,
@@ -1379,37 +1414,41 @@ def plot_validate_features_single(res_val, figsize=None, is_binary=True):
                 len(ips.flatten(res_val["pr_curve"].index)), 3, figsize=figsize
             )
         for model_name in ips.flatten(res_val["pr_curve"].index):
-            fpr = res_val["roc_curve"][model_name]["fpr"]
-            tpr = res_val["roc_curve"][model_name]["tpr"]
-            (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"]
-            mean_auc = res_val["roc_curve"][model_name]["auc"]
-
-            # Plotting
-            plot_roc_curve(
-                fpr,
-                tpr,
-                mean_auc,
-                lower_ci,
-                upper_ci,
-                model_name=model_name,
-                ax=nexttile(),
-            )
-            plot.figsets(title=model_name, sp=2)
+            try:
+                fpr = res_val["roc_curve"][model_name]["fpr"]
+                tpr = res_val["roc_curve"][model_name]["tpr"]
+                (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"]
+                mean_auc = res_val["roc_curve"][model_name]["auc"]
 
-            plot_pr_binary(
-                recall=res_val["pr_curve"][model_name]["recall"],
-                precision=res_val["pr_curve"][model_name]["precision"],
-                avg_precision=res_val["pr_curve"][model_name]["avg_precision"],
-                model_name=model_name,
-                ax=nexttile(),
-            )
-            plot.figsets(title=model_name, sp=2)
+                # Plotting
+                plot_roc_curve(
+                    fpr,
+                    tpr,
+                    mean_auc,
+                    lower_ci,
+                    upper_ci,
+                    model_name=model_name,
+                    ax=nexttile(),
+                )
+                plot.figsets(title=model_name, sp=2)
 
-            # plot cm
-            plot_cm(
-                res_val["confusion_matrix"][model_name], ax=nexttile(), normalize=False
-            )
-            plot.figsets(title=model_name, sp=2)
+                plot_pr_binary(
+                    recall=res_val["pr_curve"][model_name]["recall"],
+                    precision=res_val["pr_curve"][model_name]["precision"],
+                    avg_precision=res_val["pr_curve"][model_name]["avg_precision"],
+                    model_name=model_name,
+                    ax=nexttile(),
+                )
+                plot.figsets(title=model_name, sp=2)
+
+                # plot cm
+                plot_cm(
+                    res_val["confusion_matrix"][model_name], ax=nexttile(), normalize=False
+                )
+                plot.figsets(title=model_name, sp=2)
+                
+            except Exception as e:
+                print(e)
     else:
 
         modname_tmp = ips.flatten(res_val["roc_curve"].index)[0]
@@ -1424,22 +1463,25 @@ def plot_validate_features_single(res_val, figsize=None, is_binary=True):
         for i, model_name in enumerate(ips.flatten(res_val["pr_curve"].index)):
             ax = nexttile()
             for iclass, class_ in enumerate(classes):
-                fpr = res_val["roc_curve"][model_name]["fpr"][class_]
-                tpr = res_val["roc_curve"][model_name]["tpr"][class_]
-                (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"][iclass]
-                mean_auc = res_val["roc_curve"][model_name]["auc"][class_]
-                plot_roc_curve(
-                    fpr,
-                    tpr,
-                    mean_auc,
-                    lower_ci,
-                    upper_ci,
-                    model_name=class_,
-                    lw=1.5,
-                    color=colors[iclass],
-                    alpha=0.03,
-                    ax=ax,
-                )
+                try:
+                    fpr = res_val["roc_curve"][model_name]["fpr"][class_]
+                    tpr = res_val["roc_curve"][model_name]["tpr"][class_]
+                    (lower_ci, upper_ci) = res_val["roc_curve"][model_name]["ci95"][iclass]
+                    mean_auc = res_val["roc_curve"][model_name]["auc"][class_]
+                    plot_roc_curve(
+                        fpr,
+                        tpr,
+                        mean_auc,
+                        lower_ci,
+                        upper_ci,
+                        model_name=class_,
+                        lw=1.5,
+                        color=colors[iclass],
+                        alpha=0.03,
+                        ax=ax,
+                    )
+                except Exception as e:
+                    print(e)
             plot.figsets(
                 sp=2,
                 title=model_name,
@@ -1451,18 +1493,21 @@ def plot_validate_features_single(res_val, figsize=None, is_binary=True):
 
             ax = nexttile()
             for iclass, class_ in enumerate(classes):
-                plot_pr_curve(
-                    recall=res_val["pr_curve"][model_name]["recall"][iclass],
-                    precision=res_val["pr_curve"][model_name]["precision"][iclass],
-                    avg_precision=res_val["pr_curve"][model_name]["avg_precision"][
-                        iclass
-                    ],
-                    model_name=class_,
-                    color=colors[iclass],
-                    lw=1.5,
-                    alpha=0.03,
-                    ax=ax,
-                )
+                try:
+                    plot_pr_curve(
+                        recall=res_val["pr_curve"][model_name]["recall"][iclass],
+                        precision=res_val["pr_curve"][model_name]["precision"][iclass],
+                        avg_precision=res_val["pr_curve"][model_name]["avg_precision"][
+                            iclass
+                        ],
+                        model_name=class_,
+                        color=colors[iclass],
+                        lw=1.5,
+                        alpha=0.03,
+                        ax=ax,
+                    )
+                except Exception as e:
+                    print(e)
             plot.figsets(
                 sp=2,
                 title=class_,
@@ -1543,15 +1588,12 @@ def cal_auc_ci(
             # print("Bootstrap #{} ROC area: {:0.3f}".format(i + 1, score))
         sorted_scores = np.array(bootstrapped_scores)
         sorted_scores.sort()
-
-        # Computing the lower and upper bound of the 90% confidence interval
-        # You can change the bounds percentiles to 0.025 and 0.975 to get
-        # a 95% confidence interval instead.
+ 
         confidence_lower = sorted_scores[int((1 - ci) * len(sorted_scores))]
         confidence_upper = sorted_scores[int(ci * len(sorted_scores))]
         if verbose:
             print(
-                "Confidence interval for the score: [{:0.3f} - {:0.3}]".format(
+                "Confidence interval for the score: [{:0.3f} - {:0.3f}]".format(
                     confidence_lower, confidence_upper
                 )
             )
@@ -1568,11 +1610,8 @@ def cal_auc_ci(
             y_true, classes=np.unique(y_true)
         )  # One-vs-Rest transformation
         n_classes = y_true_bin.shape[1]  # Number of classes
-
-        bootstrapped_scores = np.zeros(
-            (n_classes, n_bootstraps)
-        )  # Store scores for each class
-
+ 
+        bootstrapped_scores = np.full((n_classes, n_bootstraps), np.nan)
         if verbose:
             print("AUROC scores for each class:")
             for i in range(n_classes):
@@ -1592,15 +1631,24 @@ def cal_auc_ci(
         # Calculating the confidence intervals for each class
         confidence_intervals = []
         for class_idx in range(n_classes):
-            sorted_scores = np.sort(bootstrapped_scores[class_idx])
-            confidence_lower = sorted_scores[int((1 - ci) * len(sorted_scores))]
-            confidence_upper = sorted_scores[int(ci * len(sorted_scores))]
-            confidence_intervals.append((confidence_lower, confidence_upper))
-
-            if verbose:
-                print(
-                    f"Class {class_idx} - Confidence interval: [{confidence_lower:.3f} - {confidence_upper:.3f}]"
-                )
+            # rm nan
+            valid_scores = bootstrapped_scores[class_idx][
+                ~np.isnan(bootstrapped_scores[class_idx])
+            ]
+            if len(valid_scores) > 0:
+                sorted_scores = np.sort(valid_scores)
+                confidence_lower = sorted_scores[int((1 - ci) * len(sorted_scores))]
+                confidence_upper = sorted_scores[int(ci * len(sorted_scores))]
+                confidence_intervals[class_idx] = (confidence_lower, confidence_upper)
+
+                if verbose:
+                    print(
+                        f"Class {class_idx} - Confidence interval: [{confidence_lower:.3f} - {confidence_upper:.3f}]"
+                    )
+            else:
+                confidence_intervals[class_idx] = (np.nan, np.nan)
+                if verbose:
+                    print(f"Class {class_idx} - Confidence interval: [NaN - NaN]") 
 
         return confidence_intervals
 
@@ -2057,20 +2105,20 @@ def rank_models(
 
         def generate_bar_plot(ax, cv_test_scores):
             ax = plot.plotxy(
-                y="Classifier", x="combined_score", data=cv_test_scores, kind="bar"
+                y="Classifier", x="combined_score", data=cv_test_scores, kind_="bar"
             )
             plt.title("Classifier Performance")
             plt.tight_layout()
             return plt
 
-        nexttile = plot.subplot(2, 2, figsize=[10, 7])
+        nexttile = plot.subplot(2, 2, figsize=[10, 10])
         generate_bar_plot(nexttile(), top_models.dropna())
         plot.radar(
             ax=nexttile(projection="polar"),
             data=cv_test_scores.set_index("Classifier"),
-            ylim=[0.5, 1],
-            color=plot.get_color(10),
-            alpha=0.05,
+            ylim=[0, 1],
+            color=plot.get_color(cv_test_scores.set_index("Classifier").shape[1]),
+            alpha=0.02,
             circular=1,
         )
     return cv_test_scores
@@ -2216,7 +2264,7 @@ def predict(
     metrics: Optional[List[str]] = None,
     stack:bool=True,# run stacking
     stacking_cv:bool=False,# stacking cross_validate, default(False),keep it simple
-    vote:bool=True,# run voting
+    vote:bool=False,# run voting
     voting:str="hard", # only for classification purporse of voting
     n_top_models:int=5, #for stacking models
     n_models_per_category:int=1, #for stacking models,可以允许同一个类别2种模型
@@ -2229,7 +2277,12 @@ def predict(
     cv_level: str = "l",  # "s":'low',"m":'medium',"l":"high"
     class_weight: str = "balanced",
     random_state: int = 1,
+    presets = "best_quality",# specific for autogluon
+    time_limit=600, # specific for autogluon
+    num_bag_folds=5, # specific for autogluon
+    num_stack_levels=2, # specific for autogluon
     verbose: bool = False,
+    **kwargs
 ) -> pd.DataFrame:
     """
     第一种情况是内部拆分，第二种是直接预测，第三种是外部验证。
@@ -2280,28 +2333,20 @@ def predict(
         RandomForestRegressor,
         ExtraTreesClassifier,
         ExtraTreesRegressor,
+        HistGradientBoostingRegressor,
         BaggingClassifier,
         BaggingRegressor,
         AdaBoostClassifier,
         AdaBoostRegressor,
     )
-    from sklearn.svm import SVC, SVR
-    from sklearn.tree import DecisionTreeRegressor
+    from sklearn.svm import SVC, SVR, LinearSVR, NuSVR
+    from sklearn.tree import DecisionTreeRegressor,ExtraTreeRegressor
     from sklearn.linear_model import (
-        LogisticRegression,
-        ElasticNet,
-        ElasticNetCV,
-        LinearRegression,
-        Lasso,
-        RidgeClassifierCV,
-        Perceptron,
-        SGDClassifier,
-        RidgeCV,
-        Ridge,
-        TheilSenRegressor,
-        HuberRegressor,
-        PoissonRegressor,
-        
+        LogisticRegression,ElasticNet,ElasticNetCV,
+        LinearRegression,Lasso,RidgeClassifierCV,Perceptron,SGDClassifier,
+        RidgeCV,Ridge,TheilSenRegressor,HuberRegressor,PoissonRegressor,Lars, LassoLars, BayesianRidge,
+        GammaRegressor, TweedieRegressor, LassoCV, LassoLarsCV, LarsCV,
+        OrthogonalMatchingPursuit, OrthogonalMatchingPursuitCV, PassiveAggressiveRegressor
     )
     from sklearn.compose import TransformedTargetRegressor
     from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
@@ -2318,10 +2363,16 @@ def predict(
     )
     from sklearn.preprocessing import PolynomialFeatures
     from sklearn.model_selection import train_test_split
-
+ 
+    from sklearn.gaussian_process import GaussianProcessRegressor
+    from sklearn.kernel_ridge import KernelRidge
+    from sklearn.dummy import DummyRegressor
+    from autogluon.tabular import TabularPredictor
     # 拼写检查
     purpose = ips.strcmp(purpose, ["classification", "regression"])[0]
     print(f"{purpose} processing...")
+
+    
     # Default models or regressors if not provided
     if purpose == "classification":
         model_ = {
@@ -2374,27 +2425,44 @@ def predict(
         model_ = {
             "Random Forest": RandomForestRegressor(random_state=random_state,n_jobs=n_jobs),
             "SVM": SVR(),  # SVR (Support Vector Regression)
-            # "Lasso": Lasso(random_state=random_state), # 它和LassoCV相同(必须要提供alpha参数),
-            "LassoCV": LassoCV(
-                cv=cv_folds, random_state=random_state,n_jobs=n_jobs
-            ),  # LassoCV自动找出最适alpha,优于Lasso
+            "LassoCV": LassoCV(cv=cv_folds, random_state=random_state,n_jobs=n_jobs),  # LassoCV自动找出最适alpha,优于Lasso
             "Gradient Boosting": GradientBoostingRegressor(random_state=random_state),
             "XGBoost": xgb.XGBRegressor(eval_metric="rmse", random_state=random_state,n_jobs=n_jobs),
             "Linear Regression": LinearRegression(n_jobs=n_jobs),
             "AdaBoost": AdaBoostRegressor(random_state=random_state),
-            "LightGBM": lgb.LGBMRegressor(random_state=random_state,n_jobs=n_jobs,                                                    
-                                          force_row_wise=True  # Or use force_col_wise=True if memory is a concern
-                                            ),
+            "LightGBM": lgb.LGBMRegressor(random_state=random_state,n_jobs=n_jobs,force_row_wise=True),  # Or use force_col_wise=True if memory is a concern
             "CatBoost": cb.CatBoostRegressor(verbose=0, random_state=random_state),
             "Extra Trees": ExtraTreesRegressor(random_state=random_state,n_jobs=n_jobs),
             "Bagging": BaggingRegressor(random_state=random_state,n_jobs=n_jobs),
             "Neural Network": MLPRegressor(max_iter=500, random_state=random_state),
             "ElasticNet": ElasticNet(random_state=random_state),
-            "Ridge": Ridge(),
+            "Ridge": Ridge(random_state=random_state),
             "KNN": KNeighborsRegressor(n_jobs=n_jobs),
             "TheilSen":TheilSenRegressor(n_jobs=n_jobs),
             "Huber":HuberRegressor(),
-            "Poisson":PoissonRegressor()
+            "Poisson":PoissonRegressor(),"LinearRegression": LinearRegression(),
+            "Lasso": Lasso(random_state=random_state),
+            "Lars": Lars(),
+            "LassoLars": LassoLars(),
+            "BayesianRidge": BayesianRidge(),
+            "GammaRegressor": GammaRegressor(),
+            "TweedieRegressor": TweedieRegressor(),
+            "LassoCV": LassoCV(random_state=random_state, n_jobs=n_jobs),
+            "ElasticNetCV": ElasticNetCV(random_state=random_state, n_jobs=n_jobs),
+            "LassoLarsCV": LassoLarsCV(n_jobs=n_jobs),
+            "LarsCV": LarsCV(),
+            "OrthogonalMatchingPursuit": OrthogonalMatchingPursuit(),
+            "OrthogonalMatchingPursuitCV": OrthogonalMatchingPursuitCV(n_jobs=n_jobs),
+            "PassiveAggressiveRegressor": PassiveAggressiveRegressor(random_state=random_state),
+            "LinearSVR": LinearSVR(random_state=random_state),
+            "NuSVR": NuSVR(),
+            "DecisionTreeRegressor": DecisionTreeRegressor(random_state=random_state),
+            "ExtraTreeRegressor": ExtraTreeRegressor(random_state=random_state),
+            "HistGradientBoostingRegressor": HistGradientBoostingRegressor(random_state=random_state),
+            "GaussianProcessRegressor": GaussianProcessRegressor(),
+            "KernelRidge": KernelRidge(),
+            "DummyRegressor": DummyRegressor(),
+            "TransformedTargetRegressor": TransformedTargetRegressor(regressor=LinearRegression())
         }
     if cls is None:
         models = model_
@@ -2411,10 +2479,17 @@ def predict(
             ips.df_special_characters_cleaner(x_true) if x_true is not None else None
         )
 
+    # only keep "autogluon_tab" in models
+    cls =  [cls] if isinstance(cls, str) else cls
+
+    if cls is not None:
+        models={"autogluon_tab":None} if "auto" in cls else models
+
     # indicate cls:
     if ips.run_once_within(30):  # 10 min
         print(f"processing: {list(models.keys())}")
-    print(isinstance(y_train, str) and y_train in x_train.columns)
+    y_train_col_name=None
+    # print(isinstance(y_train, str) and y_train in x_train.columns)
     if isinstance(y_train, str) and y_train in x_train.columns:
         y_train_col_name = y_train
         y_train = x_train[y_train]
@@ -2531,19 +2606,261 @@ def predict(
             if isinstance(y_train, np.ndarray):
                 y_true = ips.df_encoder(data=pd.DataFrame(y_true), method="label")
                 y_true = np.asarray(y_true)
+     #! so far, got the: x_train,x_true,y_train,y_true 
+    # Grid search with KFold or StratifiedKFold
+    if "autogluon_tab" in models:
+        # load hypoer_param
+        f_param = os.path.dirname(os.path.abspath(__file__))
+        f_param = f_param + "/data/hyper_param_autogluon_zeroshot2024.json"
+        with open(f_param, "r") as file:
+            hyper_param_autogluon = json.load(file)
+        # Train the model with AutoGluon
+        features=x_train.columns.tolist()
+        label= y_train_col_name if y_train_col_name is not None else 'target'
+        df_autogluon = x_train.copy()
+        df_autogluon[label]=y_train
+        autogluon_presets=["best_quality","good_quality","fast_train"]
+        best_clf = TabularPredictor(label=label, path=os.path.join(dir_save,"model_autogluon")).fit(
+            train_data=df_autogluon,
+            presets=ips.strcmp(presets, autogluon_presets)[0],  # 'best_quality' or 'good_quality' or 'fast_train'
+            time_limit=time_limit,#3600,  # in sec:  Limit training time,
+            num_bag_folds=num_bag_folds,
+            num_stack_levels=num_stack_levels,
+            hyperparameters=hyper_param_autogluon,
+            verbosity=1 if verbose else 0,
+            **kwargs
+        )
+        #! Get the leaderboard
+        gs={}
+        # Display the leaderboard for reference
+        leaderboard = best_clf.leaderboard()
+        gs['info']=best_clf.info()
+        # gs["res"]=best_clf
+        gs["features"]=features
+        gs["leaderboard"] = leaderboard
+        best_model_name = leaderboard.iloc[0, 0]  # First row, first column contains the model name
+        # Store the best model and its details in the gs dictionary
+        gs["best_estimator_"] = best_model_name  # Store the trained model, not just the name
+        gs["best_params_"] = best_model_name  # Hyperparameters
+        # Make predictions if x_true is provided
+        if x_true is not None:
+            x_true = x_true.reindex(columns=x_train.columns, fill_value=0)
+            gs["predictions"] = best_clf.predict(x_true[features],model=None)# model=None select the best 
+            gs["predict_proba"] = best_clf.predict_proba(x_true[features]) if purpose=='classification' else None
+            x_true[label]=gs["predictions"]
+            if gs["predictions"].value_counts().shape[0]>1:
+                gs['evaluate'] = best_clf.evaluate(x_true[features+[label]])
+        gs["models"]=leaderboard["model"].tolist()#best_clf.model_names() 
+        all_models = gs["models"]
+        model_evaluations = {} 
+        for model in all_models:
+            predictions = best_clf.predict(x_true[features], model=model) 
+            evaluation = best_clf.evaluate_predictions(
+                y_true=x_true[label],  # True labels
+                y_pred=predictions,    # Predictions from the specific model
+                auxiliary_metrics=True,  # Include additional metrics if needed
+            )
+            model_evaluations[model] = evaluation
+        gs["scores"]=pd.DataFrame.from_dict(model_evaluations, orient='index')
+        #! 试着保持一样的格式
+        results = {}
+        for model in all_models:
+            y_pred = best_clf.predict(x_true[features], model=model).tolist()
+            y_pred_proba=best_clf.predict_proba(x_true[features], model=model) if purpose=='classification' else None
+
+            if isinstance(y_pred_proba, pd.DataFrame):
+                y_pred_proba=y_pred_proba.iloc[:,1]
+
+            # try to make predict format consistant
+            try:
+                y_pred= [i[0] for i in y_pred]
+            except:
+                pass
+            try:
+                y_true= [i[0] for i in y_true]
+            except:
+                pass
+            try:
+                y_train= [i[0] for i in y_train]
+            except:
+                pass
+            validation_scores = {}
+            if y_true is not None and y_pred_proba is not None:
+                validation_scores = cal_metrics(
+                    y_true,
+                    y_pred,
+                    y_pred_proba=y_pred_proba,
+                    is_binary=is_binary,
+                    purpose=purpose,
+                    average="weighted",
+                )
+                if is_binary:
+                    # Calculate ROC curve
+                    # https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc.html
+                    if y_pred_proba is not None:
+                        # fpr, tpr, roc_auc = dict(), dict(), dict()
+                        fpr, tpr, _ = roc_curve(y_true, y_pred_proba)
+                        lower_ci, upper_ci = cal_auc_ci(
+                            y_true, y_pred_proba, verbose=False, is_binary=is_binary
+                        )
+                        roc_auc = auc(fpr, tpr)
+                        roc_info = {
+                            "fpr": fpr.tolist(),
+                            "tpr": tpr.tolist(),
+                            "auc": roc_auc,
+                            "ci95": (lower_ci, upper_ci),
+                        }
+                        # precision-recall curve
+                        precision_, recall_, _ = cal_precision_recall(y_true, y_pred_proba)
+                        avg_precision_ = average_precision_score(y_true, y_pred_proba)
+                        pr_info = {
+                            "precision": precision_,
+                            "recall": recall_,
+                            "avg_precision": avg_precision_,
+                        }
+                    else:
+                        roc_info, pr_info = None, None
+                    if purpose == "classification":
+                        results[model] = {
+                            # "best_clf": gs.best_estimator_,
+                            # "best_params": gs.best_params_,
+                            # "auc_indiv": [
+                            #     gs.cv_results_[f"split{i}_test_score"][gs.best_index_]
+                            #     for i in range(cv_folds)
+                            # ],
+                            "scores": validation_scores,
+                            "roc_curve": roc_info,
+                            "pr_curve": pr_info,
+                            "confusion_matrix": confusion_matrix(y_true, y_pred),
+                            "predictions": y_pred,#.tolist(),
+                            "predictions_proba": (
+                                y_pred_proba.tolist() if y_pred_proba is not None else None
+                            ),
+                            "features":features,
+                            # "coef":coef_,
+                            # "alphas":alphas_
+                        }
+                    else:  # "regression"
+                        results[model] = {
+                            # "best_clf": gs.best_estimator_,
+                            # "best_params": gs.best_params_,
+                            "scores": validation_scores,  # e.g., neg_MSE, R², etc.
+                            "predictions": y_pred,#.tolist(),
+                            "predictions_proba": (
+                                y_pred_proba.tolist() if y_pred_proba is not None else None
+                            ),
+                            "features":features,
+                            # "coef":coef_,
+                            # "alphas":alphas_
+                        }
+                else:  # multi-classes
+                    if y_pred_proba is not None:
+                        # fpr, tpr, roc_auc = dict(), dict(), dict()
+                        # fpr, tpr, _ = roc_curve(y_true, y_pred_proba)
+                        confidence_intervals = cal_auc_ci(
+                            y_true, y_pred_proba, verbose=False, is_binary=is_binary
+                        )
+                        roc_info = {
+                            "fpr": validation_scores["fpr"],
+                            "tpr": validation_scores["tpr"],
+                            "auc": validation_scores["roc_auc_by_class"],
+                            "ci95": confidence_intervals,
+                        }
+                        # precision-recall curve
+                        precision_, recall_, avg_precision_ = cal_precision_recall(
+                            y_true, y_pred_proba, is_binary=is_binary
+                        )
+                        pr_info = {
+                            "precision": precision_,
+                            "recall": recall_,
+                            "avg_precision": avg_precision_,
+                        }
+                    else:
+                        roc_info, pr_info = None, None
+
+                    if purpose == "classification":
+                        results[model] = {
+                            # "best_clf": gs.best_estimator_,
+                            # "best_params": gs.best_params_,
+                            # "auc_indiv": [
+                            #     gs.cv_results_[f"split{i}_test_score"][gs.best_index_]
+                            #     for i in range(cv_folds)
+                            # ],
+                            "scores": validation_scores,
+                            "roc_curve": roc_info,
+                            "pr_curve": pr_info,
+                            "confusion_matrix": confusion_matrix(y_true, y_pred),
+                            "predictions": y_pred,#.tolist(),
+                            "predictions_proba": (
+                                y_pred_proba.tolist() if y_pred_proba is not None else None
+                            ),
+                            "features":features,
+                            # "coef":coef_,
+                            # "alphas":alphas_
+                        }
+                    else:  # "regression"
+                        results[model] = {
+                            # "best_clf": gs.best_estimator_,
+                            # "best_params": gs.best_params_,
+                            "scores": validation_scores,  # e.g., neg_MSE, R², etc.
+                            "predictions": y_pred,#.tolist(),
+                            "predictions_proba": (
+                                y_pred_proba.tolist() if y_pred_proba is not None else None
+                            ),
+                            "features":features,
+                            # "coef":coef_,
+                            # "alphas":alphas_
+                        }
 
-    # Hyperparameter grids for tuning
-    param_grid_common_xgb = {
-                'learning_rate': [0.01, 0.1, 0.2, 0.3],
-                'max_depth': [3, 5, 7, 10],
-                'n_estimators': [50, 100, 200, 300],
-                'subsample': [0.6, 0.8, 1.0],
-                'colsample_bytree': [0.6, 0.8, 1.0],
-                'gamma': [0, 0.1, 0.2, 0.5],
-                'min_child_weight': [1, 5, 10],
-                'reg_alpha': [0, 0.1, 0.5, 1],  # L1 regularization term
-                'reg_lambda': [1, 1.5, 2],  # L2 regularization term
+            else:
+                if y_true is None:
+                    validation_scores = []
+                else:
+                    validation_scores = cal_metrics(
+                        y_true,
+                        y_pred,
+                        y_pred_proba=y_pred_proba,
+                        is_binary=is_binary,
+                        purpose=purpose,
+                        average="weighted",
+                    )
+                results[model] = {
+                    # "best_clf": gs.best_estimator_,
+                    # "best_params": gs.best_params_,
+                    "scores": validation_scores,
+                    "predictions": y_pred,#.tolist(),
+                    "predictions_proba": (
+                        y_pred_proba.tolist() if y_pred_proba is not None else None
+                    ),
+                    "features":features,
+                    "y_train": y_train if y_train is not None else [],
+                    "y_true": y_true if y_true is not None else [],
+                    # "coef":coef_,
+                    # "alphas":alphas_
                 }
+            df_results = pd.DataFrame.from_dict(results, orient="index")
+            gs['res']=df_results
+
+        if all([plot_, y_true is not None, purpose == "classification"]):
+            from datetime import datetime
+
+            now_ = datetime.now().strftime("%y%m%d_%H%M%S")
+            # try:
+            if df_results.shape[0] > 3:
+                try:
+                    plot_validate_features(df_results, is_binary=is_binary)
+                except Exception as e:
+                    print(e)
+            else:
+                try:
+                    plot_validate_features_single(df_results, is_binary=is_binary)
+                except Exception as e:
+                    print(e)
+            if dir_save:
+                ips.figsave(dir_save + f"validate_features{now_}.pdf")
+        return gs
+    
+    #! cross_valid
     if cv_level in ["low", "simple", "s", "l"]:
         param_grids = {
             "Random Forest": (
@@ -2712,7 +3029,73 @@ def predict(
                         'alpha': [0.1],
                         'max_iter': [100],},
             "Poisson":{'alpha': [0.1],
-                        'max_iter': [100],}
+                        'max_iter': [100],}, 
+            "Lars": {"n_nonzero_coefs": [10, 50, None]},
+            "LassoLars": {
+                "alpha": [0.01, 0.1, 1]
+            },
+            "BayesianRidge": {
+                "alpha_1": [1e-6, 1e-4, 1e-2],
+                "lambda_1": [1e-6, 1e-4, 1e-2]
+            },
+            "GammaRegressor": {
+                "alpha": [0.1, 1, 10]
+            },
+            "TweedieRegressor": {
+                "alpha": [0.1, 1, 10],
+                "power": [1, 1.5, 2]
+            },
+            "LassoCV": {
+                "cv": [5]
+            },
+            "ElasticNetCV": {
+                "l1_ratio": [0.2, 0.5, 0.8],
+                "cv": [5]
+            },
+            "LassoLarsCV": {
+                "cv": [5]
+            },
+            "LarsCV": {
+                "cv": [5]
+            },
+            "OrthogonalMatchingPursuit": {
+                "n_nonzero_coefs": [10, 50, None]
+            },
+            "OrthogonalMatchingPursuitCV": {
+                "cv": [5]
+            },
+            "PassiveAggressiveRegressor": {
+                "C": [0.1, 1, 10]
+            },
+            "LinearSVR": {
+                "C": [0.1, 1, 10]
+            },
+            "NuSVR": {
+                "C": [0.1, 1, 10]
+            },
+            "DecisionTreeRegressor": {
+                "max_depth": [5, 10, None]
+            },
+            "ExtraTreeRegressor": {
+                "max_depth": [5, 10, None]
+            },
+            "HistGradientBoostingRegressor": {
+                "learning_rate": [0.05, 0.1, 0.2],
+                "max_depth": [5, 10, None]
+            },
+            "GaussianProcessRegressor": {
+                "alpha": [1e-5, 1e-2, 0.1]
+            },
+            "KernelRidge": {
+                "alpha": [0.1, 1, 10],
+                "kernel": ["linear", "rbf"]
+            },
+            "DummyRegressor": {
+                "strategy": ["mean", "median"]
+            },
+            "TransformedTargetRegressor": {
+                "regressor__fit_intercept": [True, False]
+            }
         }
     elif cv_level in ["high", "advanced", "h"]:
         param_grids = {
@@ -2917,7 +3300,96 @@ def predict(
                         'alpha': [0.1, 1.0, 10.0],
                         'max_iter': [100, 200, 300],},
             "Poisson":{'alpha': [0.1, 1.0, 10.0],
-                        'max_iter': [100, 200, 300],}
+                        'max_iter': [100, 200, 300],},
+            "Lars": {
+                "n_nonzero_coefs": [10, 50, 100, 200, None]
+            },
+            "LassoLars": {
+                "alpha": [0.001, 0.01, 0.1, 1, 10]
+            },
+            "BayesianRidge": {
+                "alpha_1": [1e-6, 1e-5, 1e-4],
+                "alpha_2": [1e-6, 1e-5, 1e-4],
+                "lambda_1": [1e-6, 1e-5, 1e-4],
+                "lambda_2": [1e-6, 1e-5, 1e-4]
+            },
+            "GammaRegressor": {
+                "alpha": [0.01, 0.1, 1, 10],
+                "max_iter": [1000, 5000, 10000]
+            },
+            "TweedieRegressor": {
+                "alpha": [0.01, 0.1, 1, 10],
+                "power": [0, 1, 1.5, 2, 3]
+            },
+            "LassoCV": {
+                "alphas": [[0.001, 0.01, 0.1, 1, 10]],
+                "cv": [3, 5, 10]
+            },
+            "ElasticNetCV": {
+                "l1_ratio": [0.1, 0.5, 0.7, 0.9, 1],
+                "alphas": [[0.001, 0.01, 0.1, 1, 10]],
+                "cv": [3, 5, 10]
+            },
+            "LassoLarsCV": {
+                "cv": [3, 5, 10]
+            },
+            "LarsCV": {
+                "cv": [3, 5, 10]
+            },
+            "OrthogonalMatchingPursuit": {
+                "n_nonzero_coefs": [10, 50, 100, 200, None]
+            },
+            "OrthogonalMatchingPursuitCV": {
+                "cv": [3, 5, 10]
+            },
+            "PassiveAggressiveRegressor": {
+                "C": [0.01, 0.1, 1, 10],
+                "max_iter": [1000, 5000, 10000],
+                "early_stopping": [True, False]
+            },
+            "LinearSVR": {
+                "C": [0.01, 0.1, 1, 10],
+                "epsilon": [0.01, 0.1, 1],
+                "max_iter": [1000, 5000, 10000]
+            },
+            "NuSVR": {
+                "C": [0.01, 0.1, 1, 10],
+                "nu": [0.25, 0.5, 0.75],
+                "kernel": ["linear", "poly", "rbf", "sigmoid"]
+            },
+            "DecisionTreeRegressor": {
+                "max_depth": [None, 5, 10, 20],
+                "min_samples_split": [2, 5, 10],
+                "min_samples_leaf": [1, 2, 4]
+            },
+            "ExtraTreeRegressor": {
+                "max_depth": [None, 5, 10, 20],
+                "min_samples_split": [2, 5, 10],
+                "min_samples_leaf": [1, 2, 4]
+            },
+            "HistGradientBoostingRegressor": {
+                "learning_rate": [0.01, 0.1, 0.2],
+                "max_iter": [100, 500, 1000],
+                "max_depth": [None, 5, 10, 20],
+                "min_samples_leaf": [1, 2, 4]
+            },
+            "GaussianProcessRegressor": {
+                "alpha": [1e-10, 1e-5, 1e-2, 0.1],
+                "n_restarts_optimizer": [0, 1, 5, 10]
+            },
+            "KernelRidge": {
+                "alpha": [0.01, 0.1, 1, 10],
+                "kernel": ["linear", "poly", "rbf", "sigmoid"],
+                "degree": [2, 3, 4]
+            },
+            "DummyRegressor": {
+                "strategy": ["mean", "median", "constant"],
+                "constant": [0]  # Only if strategy is 'constant'
+            },
+            "TransformedTargetRegressor": {
+                # Grid for the underlying regressor, example shown for LinearRegression
+                "regressor__fit_intercept": [True, False]
+            }
         }
     else:  # median level
         param_grids = {
@@ -3164,7 +3636,96 @@ def predict(
                         'alpha': [0.1, 1.0],
                         'max_iter': [100, 200],},
             "Poisson":{'alpha': [0.1, 1.0],
-                        'max_iter': [100, 200],}
+                        'max_iter': [100, 200],}, 
+            "Lars": {
+                "n_nonzero_coefs": [10, 50, 100, 200, None]
+            },
+            "LassoLars": {
+                "alpha": [0.001, 0.01, 0.1, 1, 10]
+            },
+            "BayesianRidge": {
+                "alpha_1": [1e-6, 1e-5, 1e-4],
+                "alpha_2": [1e-6, 1e-5, 1e-4],
+                "lambda_1": [1e-6, 1e-5, 1e-4],
+                "lambda_2": [1e-6, 1e-5, 1e-4]
+            },
+            "GammaRegressor": {
+                "alpha": [0.01, 0.1, 1, 10],
+                "max_iter": [1000, 5000, 10000]
+            },
+            "TweedieRegressor": {
+                "alpha": [0.01, 0.1, 1, 10],
+                "power": [0, 1, 1.5, 2, 3]
+            },
+            "LassoCV": {
+                "alphas": [[0.001, 0.01, 0.1, 1, 10]],
+                "cv": [3, 5, 10]
+            },
+            "ElasticNetCV": {
+                "l1_ratio": [0.1, 0.5, 0.7, 0.9, 1],
+                "alphas": [[0.001, 0.01, 0.1, 1, 10]],
+                "cv": [3, 5, 10]
+            },
+            "LassoLarsCV": {
+                "cv": [3, 5, 10]
+            },
+            "LarsCV": {
+                "cv": [3, 5, 10]
+            },
+            "OrthogonalMatchingPursuit": {
+                "n_nonzero_coefs": [10, 50, 100, 200, None]
+            },
+            "OrthogonalMatchingPursuitCV": {
+                "cv": [3, 5, 10]
+            },
+            "PassiveAggressiveRegressor": {
+                "C": [0.01, 0.1, 1, 10],
+                "max_iter": [1000, 5000, 10000],
+                "early_stopping": [True, False]
+            },
+            "LinearSVR": {
+                "C": [0.01, 0.1, 1, 10],
+                "epsilon": [0.01, 0.1, 1],
+                "max_iter": [1000, 5000, 10000]
+            },
+            "NuSVR": {
+                "C": [0.01, 0.1, 1, 10],
+                "nu": [0.25, 0.5, 0.75],
+                "kernel": ["linear", "poly", "rbf", "sigmoid"]
+            },
+            "DecisionTreeRegressor": {
+                "max_depth": [None, 5, 10, 20],
+                "min_samples_split": [2, 5, 10],
+                "min_samples_leaf": [1, 2, 4]
+            },
+            "ExtraTreeRegressor": {
+                "max_depth": [None, 5, 10, 20],
+                "min_samples_split": [2, 5, 10],
+                "min_samples_leaf": [1, 2, 4]
+            },
+            "HistGradientBoostingRegressor": {
+                "learning_rate": [0.01, 0.1, 0.2],
+                "max_iter": [100, 500, 1000],
+                "max_depth": [None, 5, 10, 20],
+                "min_samples_leaf": [1, 2, 4]
+            },
+            "GaussianProcessRegressor": {
+                "alpha": [1e-10, 1e-5, 1e-2, 0.1],
+                "n_restarts_optimizer": [0, 1, 5, 10]
+            },
+            "KernelRidge": {
+                "alpha": [0.01, 0.1, 1, 10],
+                "kernel": ["linear", "poly", "rbf", "sigmoid"],
+                "degree": [2, 3, 4]
+            },
+            "DummyRegressor": {
+                "strategy": ["mean", "median", "constant"],
+                "constant": [0]  # Only if strategy is 'constant'
+            },
+            "TransformedTargetRegressor": {
+                # Grid for the underlying regressor, example shown for LinearRegression
+                "regressor__fit_intercept": [True, False]
+            }
         }
 
     results = {}
@@ -3174,7 +3735,7 @@ def predict(
         if purpose == "classification"
         else KFold(n_splits=cv_folds, shuffle=True, random_state=random_state)
     )
-
+ 
     # Train and validate each model
     for name, clf in tqdm(
         models.items(),
@@ -3185,8 +3746,7 @@ def predict(
         if verbose:
             print(f"\nTraining and validating {name}:")
         try:
-            # Grid search with KFold or StratifiedKFold
-            if is_binary:
+            if is_binary: 
                 gs = GridSearchCV(
                     clf,
                     param_grid=param_grids.get(name, {}),
@@ -3202,6 +3762,7 @@ def predict(
 
                 gs.fit(x_train, y_train)
                 best_clf = gs.best_estimator_
+
                 # make sure x_train and x_test has the same name
                 x_true = x_true.reindex(columns=x_train.columns, fill_value=0)
                 y_pred = best_clf.predict(x_true)
@@ -3212,7 +3773,14 @@ def predict(
                         y_pred_proba = np.hstack(
                             [1 - y_pred_proba, y_pred_proba]
                         )  # Add missing class probabilities
-                    y_pred_proba = y_pred_proba[:, 1]
+                    if y_pred_proba.shape[1] == 2:
+                        if isinstance(y_pred_proba, pd.DataFrame):
+                            y_pred_proba = y_pred_proba.iloc[:, 1] 
+                        elif isinstance(y_pred_proba, pd.Series):
+                            y_pred_proba = y_pred_proba.values[:, 1]
+                    else:
+                        y_pred_proba = y_pred_proba[:, 1]
+                    
                 elif hasattr(best_clf, "decision_function"):
                     # If predict_proba is not available, use decision_function (e.g., for SVM)
                     y_pred_proba = best_clf.decision_function(x_true)
@@ -3233,7 +3801,7 @@ def predict(
                 else: 
                     alphas_= None
                 coef_ = best_clf.coef_ if hasattr(best_clf, "coef_") else None 
-            else:
+            else: 
                 gs = GridSearchCV(
                     clf,
                     param_grid=param_grids.get(name, {}),
@@ -3289,6 +3857,7 @@ def predict(
             alphas_,coef_ = None,None
             print(f"skiped {clf}: {e}")
             continue
+
         # try to make predict format consistant
         try:
            y_pred= [i[0] for i in y_pred]
@@ -3460,7 +4029,8 @@ def predict(
 
     # Convert results to DataFrame
     df_results = pd.DataFrame.from_dict(results, orient="index")
-    # sort
+    display(df_results)
+    # sort 
     if y_true is not None:
         if purpose == "classification":
             df_scores = pd.DataFrame(
@@ -4028,6 +4598,8 @@ def predict(
         df_res=pd.concat([df_vote, df_results],ignore_index=False,axis=0)
     elif stack:
         df_res=pd.concat([df_pred,df_results],ignore_index=False,axis=0)
+    else:
+        df_res=df_results
         
     if all([plot_, y_true is not None, purpose == "classification"]):
         from datetime import datetime