py2ls 0.2.4.15__py3-none-any.whl → 0.2.4.17__py3-none-any.whl

py2ls/ml2ls.py

@@ -506,7 +506,7 @@ def get_models(
         "Support Vector Machine(svm)",
         "naive bayes",
         "Linear Discriminant Analysis (lda)",
-        "adaboost",
+        "AdaBoost",
         "DecisionTree",
         "KNeighbors",
         "Bagging",
@@ -585,7 +585,7 @@ def get_features(
         "Support Vector Machine(svm)",
         "naive bayes",
         "Linear Discriminant Analysis (lda)",
-        "adaboost",
+        "AdaBoost",
         "DecisionTree",
         "KNeighbors",
         "Bagging",
@@ -699,9 +699,11 @@ def get_features(
         "Support Vector Machine(svm)",
         "Naive Bayes",
         "Linear Discriminant Analysis (lda)",
-        "adaboost",
+        "AdaBoost",
     ]
     cls = [ips.strcmp(i, cls_)[0] for i in cls]
+    
+    feature_importances = {}
 
     # Lasso Feature Selection
     lasso_importances = (
@@ -712,6 +714,7 @@ def get_features(
     lasso_selected_features = (
         lasso_importances.head(n_features)["feature"].values if "lasso" in cls else []
     )
+    feature_importances['lasso']=lasso_importances.head(n_features)
     # Ridge
     ridge_importances = (
         features_ridge(x_train, y_train, ridge_params)
@@ -721,6 +724,7 @@ def get_features(
     selected_ridge_features = (
         ridge_importances.head(n_features)["feature"].values if "ridge" in cls else []
     )
+    feature_importances['ridge']=ridge_importances.head(n_features)
     # Elastic Net
     enet_importances = (
         features_enet(x_train, y_train, enet_params)
@@ -730,6 +734,7 @@ def get_features(
     selected_enet_features = (
         enet_importances.head(n_features)["feature"].values if "Enet" in cls else []
     )
+    feature_importances['Enet']=enet_importances.head(n_features)
     # Random Forest Feature Importance
     rf_importances = (
         features_rf(x_train, y_train, rf_params)
@@ -741,6 +746,7 @@ def get_features(
         if "Random Forest" in cls
         else []
     )
+    feature_importances['Random Forest']=rf_importances.head(n_features)
     # Gradient Boosting Feature Importance
     gb_importances = (
         features_gradient_boosting(x_train, y_train, gb_params)
@@ -752,6 +758,7 @@ def get_features(
         if "Gradient Boosting" in cls
         else []
     )
+    feature_importances['Gradient Boosting']=gb_importances.head(n_features)
     # xgb
     xgb_importances = (
         features_xgb(x_train, y_train, xgb_params) if "xgb" in cls else pd.DataFrame()
@@ -759,6 +766,7 @@ def get_features(
     top_xgb_features = (
         xgb_importances.head(n_features)["feature"].values if "xgb" in cls else []
     )
+    feature_importances['xgb']=xgb_importances.head(n_features)
 
     # SVM with RFE
     selected_svm_features = (
@@ -773,6 +781,7 @@ def get_features(
     selected_lda_features = (
         lda_importances.head(n_features)["feature"].values if "lda" in cls else []
     )
+    feature_importances['lda']=lda_importances.head(n_features)
     # AdaBoost Feature Importance
     adaboost_importances = (
         features_adaboost(x_train, y_train, adaboost_params)
@@ -784,6 +793,7 @@ def get_features(
         if "AdaBoost" in cls
         else []
     )
+    feature_importances['AdaBoost']=adaboost_importances.head(n_features)
     # Decision Tree Feature Importance
     dt_importances = (
         features_decision_tree(x_train, y_train, dt_params)
@@ -794,7 +804,8 @@ def get_features(
         dt_importances.head(n_features)["feature"].values
         if "Decision Tree" in cls
         else []
-    )
+    ) 
+    feature_importances['Decision Tree']=dt_importances.head(n_features)
     # Bagging Feature Importance
     bagging_importances = (
         features_bagging(x_train, y_train, bagging_params)
@@ -806,6 +817,7 @@ def get_features(
         if "Bagging" in cls
         else []
     )
+    feature_importances['Bagging']=bagging_importances.head(n_features)
     # KNN Feature Importance via Permutation
     knn_importances = (
         features_knn(x_train, y_train, knn_params) if "KNN" in cls else pd.DataFrame()
@@ -813,6 +825,7 @@ def get_features(
     top_knn_features = (
         knn_importances.head(n_features)["feature"].values if "KNN" in cls else []
     )
+    feature_importances['KNN']=knn_importances.head(n_features)
 
     #! Find common features
     common_features = ips.shared(
@@ -915,6 +928,7 @@ def get_features(
             "cv_train_scores": cv_train_results_df,
             "cv_test_scores": rank_models(cv_test_results_df, plot_=plot_),
             "common_features": list(common_features),
+            "feature_importances":feature_importances
         }
         if all([plot_, dir_save]):
             from datetime import datetime
@@ -927,6 +941,7 @@ def get_features(
             "cv_train_scores": pd.DataFrame(),
             "cv_test_scores": pd.DataFrame(),
             "common_features": [],
+            "feature_importances":{}
         }
         print(f"Warning: 没有找到共同的genes, when n_shared={n_shared}")
     return results
@@ -2033,6 +2048,7 @@ def predict(
     y_train: pd.Series,
     x_true: pd.DataFrame = None,
     y_true: Optional[pd.Series] = None,
+    backward:bool=False, # backward_regression 
     common_features: set = None,
     purpose: str = "classification",  # 'classification' or 'regression'
     cls: Optional[Dict[str, Any]] = None,
@@ -2227,11 +2243,21 @@ def predict(
     # else:
     #     y_train = ips.df_encoder(pd.DataFrame(y_train), method="dummy").values.ravel()
     y_train=pd.DataFrame(y_train)
-    y_train_=ips.df_encoder(y_train, method="dummy",drop=None)
-    is_binary = False if y_train_.shape[1] >2 else True
-    
-    # if is_binary:
-    #     y_train = ips.df_encoder(pd.DataFrame(y_train), method="label").values.ravel()
+    if y_train.select_dtypes(include=np.number).empty:
+        y_train_=ips.df_encoder(y_train, method="dummy",drop=None)
+        is_binary = False if y_train_.shape[1] >2 else True
+    else:
+        y_train_=ips.flatten(y_train.values)
+        is_binary = False if len(y_train_)>2 else True
+
+    if is_binary:
+        y_train = ips.df_encoder(pd.DataFrame(y_train), method="label") 
+    print('is_binary:',is_binary)
+
+    # Perform backward feature selection
+    if backward:
+        selected_features = backward_regression(x_train, y_train, threshold_out=0.05)
+        x_train=x_train[selected_features]
 
     if x_true is None:
         x_train, x_true, y_train, y_true = train_test_split(
@@ -2267,10 +2293,12 @@ def predict(
 
     # y_train=y_train.values.ravel() if y_train is not None else None
     # y_true=y_true.values.ravel() if y_true is not None else None
-    y_train = (
-        y_train.ravel() if isinstance(y_train, np.ndarray) else y_train.values.ravel()
-    )
-    y_true = y_true.ravel() if isinstance(y_true, np.ndarray) else y_true.values.ravel()
+    if y_train is not None:
+        y_train = (
+            y_train.ravel() if isinstance(y_train, np.ndarray) else y_train.values.ravel()
+        )
+    if y_true is not None:
+        y_true = y_true.ravel() if isinstance(y_true, np.ndarray) else y_true.values.ravel()
     # Ensure common features are selected
     if common_features is not None:
         x_train, x_true = x_train[common_features], x_true[common_features]
@@ -2893,7 +2921,11 @@ def predict(
             x_true = x_true.reindex(columns=x_train.columns, fill_value=0)
             y_pred = best_clf.predict(x_true)
             if hasattr(best_clf, "predict_proba"):
-                y_pred_proba = best_clf.predict_proba(x_true)[:, 1]
+                y_pred_proba = best_clf.predict_proba(x_true)
+                print("Shape of predicted probabilities:", y_pred_proba.shape)
+                if y_pred_proba.shape[1] == 1:
+                    y_pred_proba = np.hstack([1 - y_pred_proba, y_pred_proba])  # Add missing class probabilities
+                y_pred_proba = y_pred_proba[:, 1]
             elif hasattr(best_clf, "decision_function"):
                 # If predict_proba is not available, use decision_function (e.g., for SVM)
                 y_pred_proba = best_clf.decision_function(x_true)
@@ -3048,6 +3080,14 @@ def predict(
                     }
 
         else:
+            validation_scores = cal_metrics(
+                y_true,
+                y_pred,
+                y_pred_proba=y_pred_proba,
+                is_binary=is_binary,
+                purpose=purpose,
+                average="weighted",
+            )
             results[name] = {
                 "best_clf": gs.best_estimator_,
                 "best_params": gs.best_params_,
@@ -3056,6 +3096,8 @@ def predict(
                 "predictions_proba": (
                     y_pred_proba.tolist() if y_pred_proba is not None else None
                 ),
+                "y_train":y_train if y_train is not None else [],
+                "y_true": y_true if y_true is not None else []
             }
 
     # Convert results to DataFrame
@@ -3078,7 +3120,7 @@ def predict(
                 ips.figsave(dir_save + f"scores_sorted_heatmap{now_}.pdf")
             
             df_scores=df_scores.select_dtypes(include=np.number)
-            display(df_scores)
+
             if df_scores.shape[0] > 1:  # draw cluster
                 plot.heatmap(df_scores, kind="direct", cluster=True)
                 plot.figsets(xangle=30)
@@ -3169,7 +3211,14 @@ def cal_metrics(
 
         # Confusion matrix to calculate specificity
         if is_binary:
-            tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
+            cm = confusion_matrix(y_true, y_pred)
+            if cm.size == 4:
+                tn, fp, fn, tp = cm.ravel()
+            else:
+                # Handle single-class predictions
+                tn, fp, fn, tp = 0, 0, 0, 0  
+                print("Warning: Only one class found in y_pred or y_true.")
+
             # Specificity calculation
             validation_scores["specificity"] = (
                 tn / (tn + fp) if (tn + fp) > 0 else 0
@@ -3217,3 +3266,349 @@ def cal_metrics(
         )
 
     return validation_scores
+
+def plot_trees(
+    X, y, cls, max_trees=500, test_size=0.2, random_state=42, early_stopping_rounds=None
+):
+    """
+    # # Example usage:
+        # X = np.random.rand(100, 10)  # Example data with 100 samples and 10 features
+        # y = np.random.randint(0, 2, 100)  # Example binary target
+        # # Using the function with different classifiers
+        # # Random Forest example
+        # plot_trees(X, y, RandomForestClassifier(), max_trees=100)
+        # # Gradient Boosting with early stopping example
+        # plot_trees(X, y, GradientBoostingClassifier(), max_trees=100, early_stopping_rounds=10)
+        # # Extra Trees example
+        # plot_trees(X, y, ExtraTreesClassifier(), max_trees=100)
+    Master function to plot error rates (OOB, training, and testing) for different tree-based ensemble classifiers.
+
+    Parameters:
+    - X (array-like): Feature matrix.
+    - y (array-like): Target labels.
+    - cls (object): Tree-based ensemble classifier instance (e.g., RandomForestClassifier()).
+    - max_trees (int): Maximum number of trees to evaluate. Default is 500.
+    - test_size (float): Proportion of data to use as test set for testing error. Default is 0.2.
+    - random_state (int): Random state for reproducibility. Default is 42.
+    - early_stopping_rounds (int): For boosting models only, stops training if validation error doesn't improve after specified rounds.
+
+    Returns:
+    - None
+    """
+    from sklearn.model_selection import train_test_split
+    from sklearn.metrics import accuracy_score
+    from sklearn.ensemble import (
+        RandomForestClassifier,
+        BaggingClassifier,
+        ExtraTreesClassifier,
+    )
+    from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier
+    # Split data for training and testing error calculation
+    x_train, x_test, y_train, y_test = train_test_split(
+        X, y, test_size=test_size, random_state=random_state
+    )
+
+    # Initialize lists to store error rates
+    oob_error_rate = []
+    train_error_rate = []
+    test_error_rate = []
+    validation_error = None
+
+    # Configure classifier based on type
+    oob_enabled = False  # Default to no OOB error unless explicitly set
+
+    if isinstance(cls, (RandomForestClassifier, ExtraTreesClassifier)):
+        # Enable OOB if cls supports it and is using bootstrapping
+        cls.set_params(warm_start=True, n_estimators=1)
+        if hasattr(cls, "oob_score"):
+            cls.set_params(bootstrap=True, oob_score=True)
+            oob_enabled = True
+    elif isinstance(cls, BaggingClassifier):
+        cls.set_params(warm_start=True, bootstrap=True, oob_score=True, n_estimators=1)
+        oob_enabled = True
+    elif isinstance(cls, (AdaBoostClassifier, GradientBoostingClassifier)):
+        cls.set_params(n_estimators=1)
+        oob_enabled = False
+        if early_stopping_rounds:
+            validation_error = []
+
+    # Train and evaluate with an increasing number of trees
+    for i in range(1, max_trees + 1):
+        cls.set_params(n_estimators=i)
+        cls.fit(x_train, y_train)
+
+        # Calculate OOB error (for models that support it)
+        if oob_enabled and hasattr(cls, "oob_score_") and cls.oob_score:
+            oob_error = 1 - cls.oob_score_
+            oob_error_rate.append(oob_error)
+
+        # Calculate training error
+        train_error = 1 - accuracy_score(y_train, cls.predict(x_train))
+        train_error_rate.append(train_error)
+
+        # Calculate testing error
+        test_error = 1 - accuracy_score(y_test, cls.predict(x_test))
+        test_error_rate.append(test_error)
+
+        # For boosting models, use validation error with early stopping
+        if early_stopping_rounds and isinstance(
+            cls, (AdaBoostClassifier, GradientBoostingClassifier)
+        ):
+            val_error = 1 - accuracy_score(y_test, cls.predict(x_test))
+            validation_error.append(val_error)
+            if len(validation_error) > early_stopping_rounds:
+                # Stop if validation error has not improved in early_stopping_rounds
+                if validation_error[-early_stopping_rounds:] == sorted(
+                    validation_error[-early_stopping_rounds:]
+                ):
+                    print(f"Early stopping at tree {i} due to lack of improvement in validation error.")
+                    break
+
+    # Plot results
+    plt.figure(figsize=(10, 6))
+    if oob_error_rate:
+        plt.plot(
+            range(1, len(oob_error_rate) + 1),
+            oob_error_rate,
+            color="black",
+            label="OOB Error Rate",
+            linewidth=2,
+        )
+    if train_error_rate:
+        plt.plot(
+            range(1, len(train_error_rate) + 1),
+            train_error_rate,
+            linestyle="dotted",
+            color="green",
+            label="Training Error Rate",
+        )
+    if test_error_rate:
+        plt.plot(
+            range(1, len(test_error_rate) + 1),
+            test_error_rate,
+            linestyle="dashed",
+            color="red",
+            label="Testing Error Rate",
+        )
+    if validation_error:
+        plt.plot(
+            range(1, len(validation_error) + 1),
+            validation_error,
+            linestyle="solid",
+            color="blue",
+            label="Validation Error (Boosting)",
+        )
+
+    # Customize plot
+    plt.xlabel("Number of Trees")
+    plt.ylabel("Error Rate")
+    plt.title(f"Error Rate Analysis for {cls.__class__.__name__}")
+    plt.legend(loc="upper right")
+    plt.grid(True)
+    plt.show()
+
+def img_datasets_preprocessing(
+    data: pd.DataFrame,
+    x_col: str,
+    y_col: str=None,
+    target_size: tuple = (224, 224),
+    batch_size: int = 128,
+    class_mode: str = "raw",
+    shuffle: bool = False,
+    augment: bool = False,
+    scaler: str = 'normalize',  # 'normalize', 'standardize', 'clahe', 'raw'
+    grayscale: bool = False,
+    encoder: str = "label",  # Options: 'label', 'onehot', 'binary'
+    label_encoder=None,
+    kws_augmentation: dict = None,
+    verbose: bool = True,
+    drop_missing: bool = True,
+    output="df",  # "iterator":data_iterator,'df':return DataFrame
+):
+    """
+    Enhanced preprocessing function for loading and preparing image data from a DataFrame.
+
+    Parameters:
+    - df (pd.DataFrame): Input DataFrame with image paths and labels.
+    - x_col (str): Column in `df` containing image file paths.
+    - y_col (str): Column in `df` containing image labels.
+    - target_size (tuple): Desired image size in (height, width).
+    - batch_size (int): Number of images per batch.
+    - class_mode (str): Mode of label ('raw', 'categorical', 'binary').
+    - shuffle (bool): Shuffle the images in the DataFrame.
+    - augment (bool): Apply data augmentation.
+    - scaler (str): 'normalize',  # 'normalize', 'standardize', 'clahe', 'raw'
+    - grayscale (bool): Convert images to grayscale.
+    - normalize (bool): Normalize image data to [0, 1] range.
+    - encoder (str): Label encoder method ('label', 'onehot', 'binary').
+    - label_encoder: Optional pre-defined label encoder.
+    - kws_augmentation (dict): Parameters for data augmentation.
+    - verbose (bool): Print status messages.
+    - drop_missing (bool): Drop rows with missing or invalid image paths.
+
+    Returns:
+    - pd.DataFrame: DataFrame with flattened image pixels and 'Label' column.
+    """
+    from tensorflow.keras.preprocessing.image import ImageDataGenerator
+    from tensorflow.keras.utils import to_categorical
+    from sklearn.preprocessing import LabelEncoder
+    from PIL import Image
+    import os
+
+    # Validate input DataFrame for required columns
+    if y_col:
+        assert (
+            x_col in data.columns and y_col in data.columns
+        ), "Missing required columns in DataFrame."
+    if y_col is None:
+        class_mode=None
+    # 输出格式
+    output = ips.strcmp(output,[
+        "generator","tf","iterator","transform","transformer","dataframe",
+         "df","pd","pandas"])[0]
+    
+    # Handle missing file paths
+    if drop_missing:
+        data = data[
+            data[x_col].apply(lambda path: os.path.exists(path) and os.path.isfile(path))
+        ]
+
+    # Encoding labels if necessary
+    if encoder and y_col is not None:
+        if encoder == "binary":
+            data[y_col] = (data[y_col] == data[y_col].unique()[0]).astype(int)
+        elif encoder == "onehot":
+            if not label_encoder:
+                label_encoder = LabelEncoder()
+                data[y_col] = label_encoder.fit_transform(data[y_col])
+            data[y_col] = to_categorical(data[y_col])
+        elif encoder == "label":
+            if not label_encoder:
+                label_encoder = LabelEncoder()
+                data[y_col] = label_encoder.fit_transform(data[y_col])
+
+    # Set up data augmentation
+    if augment:
+        aug_params = {
+            "rotation_range": 20,
+            "width_shift_range": 0.2,
+            "height_shift_range": 0.2,
+            "shear_range": 0.2,
+            "zoom_range": 0.2,
+            "horizontal_flip": True,
+            "fill_mode": "nearest",
+        }
+        if kws_augmentation:
+            aug_params.update(kws_augmentation)
+        dat = ImageDataGenerator(rescale=scaler, **aug_params)
+        dat = ImageDataGenerator(
+        rescale=1.0 / 255 if scaler == 'normalize' else None, **aug_params)
+
+    else:
+        dat = ImageDataGenerator(
+        rescale=1.0 / 255 if scaler == 'normalize' else None)
+
+    # Create DataFrameIterator
+    data_iterator = dat.flow_from_dataframe(
+        dataframe=data,
+        x_col=x_col,
+        y_col=y_col,
+        target_size=target_size,
+        color_mode="grayscale" if grayscale else "rgb",
+        batch_size=batch_size,
+        class_mode=class_mode,
+        shuffle=shuffle,
+    )
+    print(f"target_size:{target_size}")
+    if output.lower() in ["generator", "tf", "iterator", "transform", "transformer"]:
+        return data_iterator
+    elif output.lower() in ["dataframe", "df", "pd", "pandas"]:
+        # Initialize list to collect processed data
+        data_list = []
+        total_batches = data_iterator.n // batch_size
+
+        # Load, resize, and process images in batches
+        for i, (batch_images, batch_labels) in enumerate(data_iterator):
+            for img, label in zip(batch_images, batch_labels): 
+                if scaler == ['normalize','raw']:
+                    # Already rescaled by 1.0/255 in ImageDataGenerator
+                    pass
+                elif scaler == 'standardize':
+                    # Standardize by subtracting mean and dividing by std
+                    img = (img - np.mean(img)) / np.std(img)
+                elif scaler == 'clahe':
+                    # Apply CLAHE to the image
+                    img = apply_clahe(img)
+                flat_img = img.flatten()
+                data_list.append(np.append(flat_img, label))
+
+            # Stop when all images have been processed
+            if i >= total_batches:
+                break
+
+        # Define column names for flattened image data
+        pixel_count = target_size[0] * target_size[1] * (1 if grayscale else 3)
+        column_names = [f"pixel_{i}" for i in range(pixel_count)] + ["Label"]
+
+        # Create DataFrame from flattened data
+        df_img = pd.DataFrame(data_list, columns=column_names)
+
+        if verbose:
+            print("Processed images:", len(df_img))
+            print("Final DataFrame shape:", df_img.shape)
+            display(df_img.head())
+
+        return df_img
+
+
+def backward_regression(X:pd.DataFrame, y:pd.Series, initial_list=[], threshold_out=0.05, verbose=True):
+    """
+    # awesome bit of code from https://www.kaggle.com/code/adibouayjan/house-price-step-by-step-modeling
+    
+    Evaluates the p-values of all features, which represent the probability of observing a coefficient 
+    as extreme as the one calculated if the feature had no true effect on the target.
+
+    Args:
+        X -- features values
+        y -- target variable
+        initial_list -- features header
+        threshold_out -- pvalue threshold of features to drop
+        verbose -- true to produce lots of logging output
+
+    Returns:
+        list of selected features for modeling 
+    """
+    import statsmodels.api as sm
+    if isinstance(y, str) and y in X.columns:
+        y_col_name = y
+        y = X[y]
+        X = X.drop(y_col_name, axis=1)
+    included = list(X.columns)
+    while True:
+        changed = False
+        model = sm.OLS(y, sm.add_constant(pd.DataFrame(X[included]))).fit()
+        # exclude the intercept for p-value checking
+        pvalues = model.pvalues.iloc[1:]
+        worst_pval = pvalues.max()
+        if worst_pval > threshold_out:
+            changed = True
+            worst_feature = pvalues.idxmax()
+            included.remove(worst_feature)
+            if verbose:
+                print(f"Removing feature '{worst_feature}' with p-value {worst_pval}")
+        if not changed:
+            break
+    print(f"\nSelected Features:\n{included}")
+    return included  # Returns the list of selected features
+ 
+
+# Function to apply CLAHE (Contrast Limited Adaptive Histogram Equalization)
+def apply_clahe(img):
+    import cv2
+    lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)  # Convert to LAB color space
+    l, a, b = cv2.split(lab)  # Split into channels
+    clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8))
+    cl = clahe.apply(l)  # Apply CLAHE to the L channel
+    limg = cv2.merge((cl, a, b))  # Merge back the channels
+    img_clahe = cv2.cvtColor(limg, cv2.COLOR_LAB2RGB)  # Convert back to RGB
+    return img_clahe