py2ls 0.2.4.16__tar.gz → 0.2.4.17__tar.gz

{py2ls-0.2.4.16 → py2ls-0.2.4.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: py2ls
-Version: 0.2.4.16
+Version: 0.2.4.17
 Summary: py(thon)2(too)ls
 Author: Jianfeng
 Author-email: Jianfeng.Liu0413@gmail.com
@@ -200,6 +200,7 @@ Requires-Dist: selenium (>=4.23.1)
 Requires-Dist: setuptools (>=70.3.0)
 Requires-Dist: shellingham (>=1.5.4)
 Requires-Dist: six (>=1.16.0)
+Requires-Dist: skorch (>=1.0.0)
 Requires-Dist: sniffio (>=1.3.1)
 Requires-Dist: sortedcontainers (>=2.4.0)
 Requires-Dist: soupsieve (>=2.5)

{py2ls-0.2.4.16 → py2ls-0.2.4.17}/py2ls/ips.py

@@ -2171,6 +2171,8 @@ def fload(fpath, kind=None, **kwargs):
                                     continue
                             else:
                                 pass
+        if is_df_abnormal(df,verbose=verbose):
+            df=pd.read_csv(fpath,**kwargs)
         display(df.head(2))
         print(f"shape: {df.shape}")
         return df

{py2ls-0.2.4.16 → py2ls-0.2.4.17}/py2ls/ml2ls.py

@@ -2048,6 +2048,7 @@ def predict(
     y_train: pd.Series,
     x_true: pd.DataFrame = None,
     y_true: Optional[pd.Series] = None,
+    backward:bool=False, # backward_regression 
     common_features: set = None,
     purpose: str = "classification",  # 'classification' or 'regression'
     cls: Optional[Dict[str, Any]] = None,
@@ -2252,6 +2253,12 @@ def predict(
     if is_binary:
         y_train = ips.df_encoder(pd.DataFrame(y_train), method="label") 
     print('is_binary:',is_binary)
+
+    # Perform backward feature selection
+    if backward:
+        selected_features = backward_regression(x_train, y_train, threshold_out=0.05)
+        x_train=x_train[selected_features]
+
     if x_true is None:
         x_train, x_true, y_train, y_true = train_test_split(
             x_train,
@@ -2286,10 +2293,12 @@ def predict(
 
     # y_train=y_train.values.ravel() if y_train is not None else None
     # y_true=y_true.values.ravel() if y_true is not None else None
-    y_train = (
-        y_train.ravel() if isinstance(y_train, np.ndarray) else y_train.values.ravel()
-    )
-    y_true = y_true.ravel() if isinstance(y_true, np.ndarray) else y_true.values.ravel()
+    if y_train is not None:
+        y_train = (
+            y_train.ravel() if isinstance(y_train, np.ndarray) else y_train.values.ravel()
+        )
+    if y_true is not None:
+        y_true = y_true.ravel() if isinstance(y_true, np.ndarray) else y_true.values.ravel()
     # Ensure common features are selected
     if common_features is not None:
         x_train, x_true = x_train[common_features], x_true[common_features]
@@ -3071,6 +3080,14 @@ def predict(
                     }
 
         else:
+            validation_scores = cal_metrics(
+                y_true,
+                y_pred,
+                y_pred_proba=y_pred_proba,
+                is_binary=is_binary,
+                purpose=purpose,
+                average="weighted",
+            )
             results[name] = {
                 "best_clf": gs.best_estimator_,
                 "best_params": gs.best_params_,
@@ -3079,6 +3096,8 @@ def predict(
                 "predictions_proba": (
                     y_pred_proba.tolist() if y_pred_proba is not None else None
                 ),
+                "y_train":y_train if y_train is not None else [],
+                "y_true": y_true if y_true is not None else []
             }
 
     # Convert results to DataFrame
@@ -3540,6 +3559,49 @@ def img_datasets_preprocessing(
             display(df_img.head())
 
         return df_img
+
+
+def backward_regression(X:pd.DataFrame, y:pd.Series, initial_list=[], threshold_out=0.05, verbose=True):
+    """
+    # awesome bit of code from https://www.kaggle.com/code/adibouayjan/house-price-step-by-step-modeling
+    
+    Evaluates the p-values of all features, which represent the probability of observing a coefficient 
+    as extreme as the one calculated if the feature had no true effect on the target.
+
+    Args:
+        X -- features values
+        y -- target variable
+        initial_list -- features header
+        threshold_out -- pvalue threshold of features to drop
+        verbose -- true to produce lots of logging output
+
+    Returns:
+        list of selected features for modeling 
+    """
+    import statsmodels.api as sm
+    if isinstance(y, str) and y in X.columns:
+        y_col_name = y
+        y = X[y]
+        X = X.drop(y_col_name, axis=1)
+    included = list(X.columns)
+    while True:
+        changed = False
+        model = sm.OLS(y, sm.add_constant(pd.DataFrame(X[included]))).fit()
+        # exclude the intercept for p-value checking
+        pvalues = model.pvalues.iloc[1:]
+        worst_pval = pvalues.max()
+        if worst_pval > threshold_out:
+            changed = True
+            worst_feature = pvalues.idxmax()
+            included.remove(worst_feature)
+            if verbose:
+                print(f"Removing feature '{worst_feature}' with p-value {worst_pval}")
+        if not changed:
+            break
+    print(f"\nSelected Features:\n{included}")
+    return included  # Returns the list of selected features
+ 
+
 # Function to apply CLAHE (Contrast Limited Adaptive Histogram Equalization)
 def apply_clahe(img):
     import cv2

{py2ls-0.2.4.16 → py2ls-0.2.4.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "py2ls"
-version = "0.2.4.16"
+version = "0.2.4.17"
 description = "py(thon)2(too)ls"
 authors = ["Jianfeng <Jianfeng.Liu0413@gmail.com>"]
 readme = "README.md"
@@ -197,6 +197,7 @@ selenium = ">=4.23.1"
 setuptools = ">=70.3.0"
 shellingham = ">=1.5.4"
 six = ">=1.16.0"
+skorch =  ">=1.0.0"
 sniffio = ">=1.3.1"
 sortedcontainers = ">=2.4.0"
 soupsieve = ">=2.5"