gitlabds 2.4.0__tar.gz → 2.4.1__tar.gz

{gitlabds-2.4.0 → gitlabds-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gitlabds
-Version: 2.4.0
+Version: 2.4.1
 Summary: GitLab Data Science Tools
 Author-email: Kevin Dietz <kdietz@gitlab.com>
 License-Expression: MIT
@@ -376,9 +376,9 @@ test_df_transformed = gitlabds.apply_dummy(
 <details><summary> Remove Low Variation </summary>
 
 #### Description
-Remove columns from a dataset that do not meet the variation threshold. That is, columns will be dropped that contain a high percentage of one value.
+Remove columns from a dataset that do not meet the variation threshold. That is, columns will be dropped that contain a high percentage of one value. Columns that are entirely null are always dropped.
 
-`gitlabds.remove_low_variation(df=None, dv=None, columns='all', threshold=.98, verbose=True):`
+`gitlabds.remove_low_variation(df=None, dv=None, columns='all', threshold=.98, verbose=True, dropna=False):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -386,6 +386,7 @@ Remove columns from a dataset that do not meet the variation threshold. That is,
 - **_columns_** : Will examine at all columns by default. To limit to just a subset of columns, pass a list of column names. 
 - **_threshold_**: The maximum percentage one value in a column can represent. columns that exceed this threshold will be dropped. For example, the default value of `0.98` will drop any column where one value is present in more than 98% of rows.
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `False` (default) to count missing values as their own value. Set to `True` to measure variation over non-null values only, so a column that is one value plus NaN is dropped. Use `True` when missing values are left in place for a model that handles them natively (e.g. XGBoost, LightGBM).
 
 #### Returns
 - DataFrame with low variation columns dropped.
@@ -399,6 +400,10 @@ new_df = gitlabds.remove_low_variation(df=my_df, dv='my_outcome', columns='all',
 # Drop any of the selected columns where one value is present in more than 99% of rows.
 new_df = gitlabds.remove_low_variation(df=my_df, dv=None, columns=['colA', 'colB', 'colC'], threshold=.99)
 ```
+```python
+# Ignore missing values: drop columns where one value makes up more than 98% of the non-null rows.
+new_df = gitlabds.remove_low_variation(df=my_df, dv='my_outcome', dropna=True)
+```
 </details>
 
 <details><summary> Correlation Reduction </summary>
@@ -406,7 +411,7 @@ new_df = gitlabds.remove_low_variation(df=my_df, dv=None, columns=['colA', 'colB
 #### Description
 Reduce the number of columns on a dataframe by dropping columns that are highly correlated with other columns. Note: only one of the two highly correlated columns will be dropped.
 
-`gitlabds.correlation_reduction(df=None, dv=None, threshold=0.9, method="pearson", verbose=True):`
+`gitlabds.correlation_reduction(df=None, dv=None, threshold=0.9, method="pearson", verbose=True, dropna=True, min_periods=1):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -414,6 +419,8 @@ Reduce the number of columns on a dataframe by dropping columns that are highly
 - **_threshold_**: The threshold above which columns will be dropped. If two variables exceed this threshold, one will be dropped from the dataframe. For example, the default value of `0.90` will identify columns that have correlations greater than 90% to each other and drop one of those columns.
 - **_method_**: The correlation method to use. Options are "pearson" (linear relationships), "spearman" (monotonic relationships), or "mutual_info" (any statistical dependency).
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `True` (default) to compare each pair of columns (and each column with the outcome) using only rows where both are non-null. Pearson and Spearman always work this way; for "mutual_info", `False` raises an error if any missing values are present.
+- **_min_periods_** : Minimum number of rows where both columns of a pair are non-null for the pair to be evaluated. Pairs below it are never treated as correlated. Guards against spurious high correlations between sparsely populated columns. "mutual_info" always requires at least 4 such rows.
 
 #### Returns
 - DataFrame with redundant correlated columns dropped.
@@ -434,7 +441,7 @@ new_df = gitlabds.correlation_reduction(df=my_df, dv=None, threshold=0.90, metho
 #### Description
 Remove columns that are highly correlated with the outcome (target) column.
 
-`gitlabds.remove_outcome_proxies(df, dv, threshold=.8, method="pearson", verbose=True):`
+`gitlabds.remove_outcome_proxies(df, dv, threshold=.8, method="pearson", verbose=True, dropna=True, min_periods=1):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -442,6 +449,8 @@ Remove columns that are highly correlated with the outcome (target) column.
 - _**threshold**_ : The correlation value to the outcome above which columns will be dropped. For example, the default value of `0.80` will identify and drop columns that have correlations greater than 80% to the outcome.
 - **_method_**: The correlation method to use. Options are "pearson" (linear relationships), "spearman" (monotonic relationships), or "mutual_info" (any statistical dependency).
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `True` (default) to compare each column to the outcome using only rows where both are non-null. Pearson and Spearman always work this way; for "mutual_info", `False` raises an error if any missing values are present.
+- **_min_periods_** : Minimum number of rows where both the column and the outcome are non-null for the column to be evaluated. Columns below it are never dropped. Guards against spurious high correlations from sparsely populated columns. "mutual_info" always requires at least 4 such rows.
 
 #### Returns
 - DataFrame with outcome proxy columns dropped.
@@ -455,6 +464,10 @@ new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', threshold=.7
 # Drop columns with correlations to the outcome greater than 80% using Spearman correlation
 new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', threshold=.8, method="spearman")        
 ```
+```python
+# Data with missing values: only evaluate columns with at least 100 non-null rows alongside the outcome
+new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', method="spearman", min_periods=100)
+```
 </details>
 
 <details><summary> Drop Categorical </summary>

{gitlabds-2.4.0 → gitlabds-2.4.1}/README.md

@@ -340,9 +340,9 @@ test_df_transformed = gitlabds.apply_dummy(
 <details><summary> Remove Low Variation </summary>
 
 #### Description
-Remove columns from a dataset that do not meet the variation threshold. That is, columns will be dropped that contain a high percentage of one value.
+Remove columns from a dataset that do not meet the variation threshold. That is, columns will be dropped that contain a high percentage of one value. Columns that are entirely null are always dropped.
 
-`gitlabds.remove_low_variation(df=None, dv=None, columns='all', threshold=.98, verbose=True):`
+`gitlabds.remove_low_variation(df=None, dv=None, columns='all', threshold=.98, verbose=True, dropna=False):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -350,6 +350,7 @@ Remove columns from a dataset that do not meet the variation threshold. That is,
 - **_columns_** : Will examine at all columns by default. To limit to just a subset of columns, pass a list of column names. 
 - **_threshold_**: The maximum percentage one value in a column can represent. columns that exceed this threshold will be dropped. For example, the default value of `0.98` will drop any column where one value is present in more than 98% of rows.
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `False` (default) to count missing values as their own value. Set to `True` to measure variation over non-null values only, so a column that is one value plus NaN is dropped. Use `True` when missing values are left in place for a model that handles them natively (e.g. XGBoost, LightGBM).
 
 #### Returns
 - DataFrame with low variation columns dropped.
@@ -363,6 +364,10 @@ new_df = gitlabds.remove_low_variation(df=my_df, dv='my_outcome', columns='all',
 # Drop any of the selected columns where one value is present in more than 99% of rows.
 new_df = gitlabds.remove_low_variation(df=my_df, dv=None, columns=['colA', 'colB', 'colC'], threshold=.99)
 ```
+```python
+# Ignore missing values: drop columns where one value makes up more than 98% of the non-null rows.
+new_df = gitlabds.remove_low_variation(df=my_df, dv='my_outcome', dropna=True)
+```
 </details>
 
 <details><summary> Correlation Reduction </summary>
@@ -370,7 +375,7 @@ new_df = gitlabds.remove_low_variation(df=my_df, dv=None, columns=['colA', 'colB
 #### Description
 Reduce the number of columns on a dataframe by dropping columns that are highly correlated with other columns. Note: only one of the two highly correlated columns will be dropped.
 
-`gitlabds.correlation_reduction(df=None, dv=None, threshold=0.9, method="pearson", verbose=True):`
+`gitlabds.correlation_reduction(df=None, dv=None, threshold=0.9, method="pearson", verbose=True, dropna=True, min_periods=1):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -378,6 +383,8 @@ Reduce the number of columns on a dataframe by dropping columns that are highly
 - **_threshold_**: The threshold above which columns will be dropped. If two variables exceed this threshold, one will be dropped from the dataframe. For example, the default value of `0.90` will identify columns that have correlations greater than 90% to each other and drop one of those columns.
 - **_method_**: The correlation method to use. Options are "pearson" (linear relationships), "spearman" (monotonic relationships), or "mutual_info" (any statistical dependency).
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `True` (default) to compare each pair of columns (and each column with the outcome) using only rows where both are non-null. Pearson and Spearman always work this way; for "mutual_info", `False` raises an error if any missing values are present.
+- **_min_periods_** : Minimum number of rows where both columns of a pair are non-null for the pair to be evaluated. Pairs below it are never treated as correlated. Guards against spurious high correlations between sparsely populated columns. "mutual_info" always requires at least 4 such rows.
 
 #### Returns
 - DataFrame with redundant correlated columns dropped.
@@ -398,7 +405,7 @@ new_df = gitlabds.correlation_reduction(df=my_df, dv=None, threshold=0.90, metho
 #### Description
 Remove columns that are highly correlated with the outcome (target) column.
 
-`gitlabds.remove_outcome_proxies(df, dv, threshold=.8, method="pearson", verbose=True):`
+`gitlabds.remove_outcome_proxies(df, dv, threshold=.8, method="pearson", verbose=True, dropna=True, min_periods=1):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -406,6 +413,8 @@ Remove columns that are highly correlated with the outcome (target) column.
 - _**threshold**_ : The correlation value to the outcome above which columns will be dropped. For example, the default value of `0.80` will identify and drop columns that have correlations greater than 80% to the outcome.
 - **_method_**: The correlation method to use. Options are "pearson" (linear relationships), "spearman" (monotonic relationships), or "mutual_info" (any statistical dependency).
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `True` (default) to compare each column to the outcome using only rows where both are non-null. Pearson and Spearman always work this way; for "mutual_info", `False` raises an error if any missing values are present.
+- **_min_periods_** : Minimum number of rows where both the column and the outcome are non-null for the column to be evaluated. Columns below it are never dropped. Guards against spurious high correlations from sparsely populated columns. "mutual_info" always requires at least 4 such rows.
 
 #### Returns
 - DataFrame with outcome proxy columns dropped.
@@ -419,6 +428,10 @@ new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', threshold=.7
 # Drop columns with correlations to the outcome greater than 80% using Spearman correlation
 new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', threshold=.8, method="spearman")        
 ```
+```python
+# Data with missing values: only evaluate columns with at least 100 non-null rows alongside the outcome
+new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', method="spearman", min_periods=100)
+```
 </details>
 
 <details><summary> Drop Categorical </summary>

{gitlabds-2.4.0 → gitlabds-2.4.1}/gitlabds/feature_reduction.py

@@ -57,13 +57,15 @@ def remove_low_variation(
     dv: Optional[str] = None,
     columns: Union[str, List[str]] = "all",
     threshold: float = 0.98,
-    verbose: bool = True
+    verbose: bool = True,
+    dropna: bool = False
 ) -> pd.DataFrame:
     """
     Remove columns from a dataset that do not meet the variation threshold.
     
     Columns will be dropped if they contain a high percentage of one value, as they
-    provide minimal information for modeling.
+    provide minimal information for modeling. Columns that are entirely null are
+    always dropped.
     
     Parameters
     ----------
@@ -81,6 +83,13 @@ def remove_low_variation(
         will drop any column where one value is present in more than 98% of rows.
     verbose : bool, default=True
         Set to True to print outputs of columns being dropped. Set to False to suppress.
+    dropna : bool, default=False
+        If False, missing values count as their own value, so a column that is one value
+        plus NaN is kept as long as neither exceeds the threshold. If True, the share of
+        the most common value is computed over non-null rows only, so a column with a
+        single distinct non-null value is dropped regardless of how many NaNs it has.
+        Use True when missing values will be left in place for a model that handles them
+        natively (e.g. XGBoost, LightGBM).
     
     Returns
     -------
@@ -90,7 +99,7 @@ def remove_low_variation(
     Examples
     --------
     >>> new_df = remove_low_variation(df=my_df, dv='target', threshold=0.95)
-    >>> print(f"Removed columns due to low variation")
+    >>> new_df = remove_low_variation(df=my_df, dv='target', dropna=True)
     """
     # Enhanced parameter validation
     if not isinstance(df, pd.DataFrame):
@@ -105,6 +114,9 @@ def remove_low_variation(
     if dv is not None and dv not in df.columns:
         warnings.warn(f"Specified DV '{dv}' not found in dataframe")
     
+    if not isinstance(dropna, bool):
+        raise TypeError(f"dropna must be a bool, got {type(dropna)}")
+    
     # Create a deep copy of the dataframe
     work_df = df.copy(deep=True)
     
@@ -115,7 +127,7 @@ def remove_low_variation(
             print("\nExamining all columns as candidates for removal")
     else:
         if isinstance(columns, list):
-            var_list = columns
+            var_list = list(columns)
             if verbose:
                 print(f"\nExamining the following columns as candidates for removal: {var_list}")
         else:
@@ -139,12 +151,15 @@ def remove_low_variation(
     # Process all columns
     for v in var_list:
         try:
-            # Skip non-finite columns
+            # Drop columns with no non-null values
             if work_df[v].isna().all():
+                removal_list.append(v)
+                if verbose:
+                    print(f"Column '{v}' contains only null values")
                 continue
                 
             # Get value counts for the column
-            counts = work_df[v].value_counts(normalize=True, dropna=False)
+            counts = work_df[v].value_counts(normalize=True, dropna=dropna)
             max_freq = counts.iloc[0]
             max_value = counts.index[0]
             
@@ -153,7 +168,8 @@ def remove_low_variation(
                 removal_list.append(v)
                 
                 if verbose:
-                    print(f"In column '{v}', the value {max_value} accounts for {max_freq:.2%} of the values (threshold: {threshold:.2%})")
+                    basis = "non-null values" if dropna else "values"
+                    print(f"In column '{v}', the value {max_value} accounts for {max_freq:.2%} of the {basis} (threshold: {threshold:.2%})")
         except Exception as e:
             warnings.warn(f"Error processing column '{v}': {str(e)}")
     
@@ -166,12 +182,47 @@ def remove_low_variation(
     return work_df
 
 
+def _mutual_info_to_target(
+    X: pd.DataFrame,
+    y: pd.Series,
+    min_periods: int = 1
+) -> pd.Series:
+    """
+    Mutual information of each column of X with y, on rows where both are non-null.
+
+    Columns with no nulls on y's non-null rows are scored in one call; the rest are scored
+    one at a time on their own complete rows. Columns with fewer than max(min_periods, 4)
+    complete rows are returned as NaN.
+    """
+    if isinstance(y.dtype, pd.CategoricalDtype) or y.nunique() < 10:
+        mi_func = mutual_info_classif
+    else:
+        mi_func = mutual_info_regression
+    
+    y_rows = y.notna()
+    pair_counts = X.notna().mul(y_rows, axis=0).sum()
+    eligible = pair_counts[pair_counts >= max(min_periods, 4)]
+    complete_cols = eligible.index[eligible == y_rows.sum()].tolist()
+    partial_cols = eligible.index[eligible < y_rows.sum()].tolist()
+    
+    mi_values = pd.Series(np.nan, index=X.columns, dtype=float)
+    if complete_cols:
+        mi_values[complete_cols] = mi_func(X.loc[y_rows, complete_cols], y[y_rows])
+    for col in partial_cols:
+        rows = y_rows & X[col].notna()
+        mi_values[col] = mi_func(X.loc[rows, [col]], y[rows])[0]
+    
+    return mi_values
+
+
 def remove_outcome_proxies(
     df: pd.DataFrame,
     dv: str,
     threshold: float = 0.8,
     method: str = "pearson",
-    verbose: bool = True
+    verbose: bool = True,
+    dropna: bool = True,
+    min_periods: int = 1
 ) -> pd.DataFrame:
     """
     Remove columns that are highly correlated with the outcome (target) column.
@@ -196,6 +247,17 @@ def remove_outcome_proxies(
         - "mutual_info": Mutual information (any statistical dependency)
     verbose : bool, default=True
         Set to True to print outputs of columns being dropped. Set to False to suppress.
+    dropna : bool, default=True
+        If True, each feature is compared to `dv` using only the rows where both are
+        non-null. "pearson" and "spearman" always work this way (pandas pairwise-complete
+        correlation), so this flag only changes "mutual_info": with False, any missing
+        value in the numeric features or `dv` raises a ValueError, because scikit-learn's
+        mutual information estimators do not accept NaN.
+    min_periods : int, default=1
+        Minimum number of rows where both the feature and `dv` are non-null for the
+        feature to be evaluated. Features below it are never flagged as proxies. Guards
+        against spurious high correlations from sparsely populated columns. "mutual_info"
+        always requires at least 4 such rows.
     
     Returns
     -------
@@ -204,8 +266,8 @@ def remove_outcome_proxies(
     
     Examples
     --------
-    >>> new_df = find_outcome_proxies(df=my_df, dv='target', threshold=0.7, method="spearman")
-    >>> print(f"Removed columns as outcome proxies")
+    >>> new_df = remove_outcome_proxies(df=my_df, dv='target', threshold=0.7, method="spearman")
+    >>> new_df = remove_outcome_proxies(df=my_df, dv='target', method="mutual_info", min_periods=100)
     """
     # Parameter validation
     if not isinstance(df, pd.DataFrame):
@@ -230,6 +292,15 @@ def remove_outcome_proxies(
     if method not in valid_methods:
         raise ValueError(f"method must be one of {valid_methods}, got {method}")
     
+    if not isinstance(dropna, bool):
+        raise TypeError(f"dropna must be a bool, got {type(dropna)}")
+    
+    if isinstance(min_periods, bool) or not isinstance(min_periods, int):
+        raise TypeError(f"min_periods must be an int, got {type(min_periods)}")
+    
+    if min_periods < 1:
+        raise ValueError(f"min_periods must be at least 1, got {min_periods}")
+    
     # Create a deep copy of the dataframe
     work_df = df.copy(deep=True)
     
@@ -241,35 +312,35 @@ def remove_outcome_proxies(
     # Remove the target variable from the list of features
     feature_cols = [col for col in numeric_cols if col != dv]
     
+    y = work_df[dv]
+    
     # Calculate correlations based on the specified method
     if method in ["pearson", "spearman"]:
-        # Calculate correlations using pandas
-        correlations = work_df[numeric_cols].corr(method=method)[dv].drop(dv)
+        # Per-feature correlation with the dv (what DataFrame.corrwith does, plus min_periods);
+        # avoids computing the full feature-by-feature matrix
+        correlations = pd.Series(
+            {col: work_df[col].corr(y, method=method, min_periods=min_periods) for col in feature_cols},
+            dtype=float
+        )
         corr_values = {col: abs(corr) for col, corr in correlations.items() if abs(corr) > threshold}
     
     elif method == "mutual_info":
-        # For mutual information, we need to handle the target type differently
-        # Create X and y for mutual information calculation
         X = work_df[feature_cols]
-        y = work_df[dv]
         
-        # Determine if this is a classification or regression problem
-        if pd.api.types.is_categorical_dtype(y) or y.nunique() < 10:
-            # Classification problem
-            mi_values = mutual_info_classif(X, y)
-        else:
-            # Regression problem
-            mi_values = mutual_info_regression(X, y)
+        if not dropna and (X.isna().any().any() or y.isna().any()):
+            raise ValueError(
+                "mutual_info cannot be computed on missing values; set dropna=True to use "
+                "the rows where each feature and the dv are both non-null"
+            )
+        
+        mi_values = _mutual_info_to_target(X, y, min_periods)
         
         # Normalize MI values to 0-1 range for comparison with threshold
-        if len(mi_values) > 0:
-            mi_max = max(mi_values) if max(mi_values) > 0 else 1
-            mi_normalized = mi_values / mi_max
-            
-            # Create dictionary of feature: mi_value
-            corr_values = {col: mi for col, mi in zip(feature_cols, mi_normalized) if mi > threshold}
-        else:
-            corr_values = {}
+        mi_max = mi_values.max()
+        if not mi_max > 0:
+            mi_max = 1
+        mi_normalized = mi_values / mi_max
+        corr_values = {col: mi for col, mi in mi_normalized.items() if mi > threshold}
     
     # Create the list of columns to drop
     dropped_columns = list(corr_values.keys())
@@ -295,7 +366,9 @@ def correlation_reduction(
     dv: Optional[str] = None,
     threshold: float = 0.9,
     method: str = "pearson",
-    verbose: bool = True
+    verbose: bool = True,
+    dropna: bool = True,
+    min_periods: int = 1
 ) -> pd.DataFrame:
     """
     Reduce the number of columns by dropping features that are highly correlated with each other.
@@ -311,6 +384,7 @@ def correlation_reduction(
     dv : str, optional
         The column name of your outcome. If provided, when choosing between 
         correlated features, the one with higher correlation to the target will be kept.
+        A feature whose correlation to the target cannot be computed counts as 0.
     threshold : float, default=0.9
         The correlation threshold above which columns will be considered redundant.
         For example, the default value of 0.9 will identify columns that have
@@ -319,9 +393,21 @@ def correlation_reduction(
         The correlation method to use. Options are:
         - "pearson": Standard correlation coefficient (linear relationships)
         - "spearman": Spearman rank correlation (monotonic relationships)
-        - "mutual_info": Mutual information (any statistical dependency)
+        - "mutual_info": Mutual information (any statistical dependency). Scores every
+          feature pair, so it is slow on wide data.
     verbose : bool, default=True
         Set to True to print outputs of columns being dropped. Set to False to suppress.
+    dropna : bool, default=True
+        If True, each pair (feature-feature, and feature-dv) is compared using only the
+        rows where both are non-null. "pearson" and "spearman" always work this way
+        (pandas pairwise-complete correlation), so this flag only changes "mutual_info":
+        with False, any missing value in the numeric features or `dv` raises a ValueError,
+        because scikit-learn's mutual information estimators do not accept NaN.
+    min_periods : int, default=1
+        Minimum number of rows where both columns of a pair are non-null for the pair to
+        be evaluated. Pairs below it are never treated as correlated. Guards against
+        spurious high correlations between sparsely populated columns. "mutual_info"
+        always requires at least 4 such rows.
     
     Returns
     -------
@@ -330,8 +416,8 @@ def correlation_reduction(
     
     Examples
     --------
-    >>> new_df = reduce_correlation(df=my_df, dv='target', threshold=0.85, method="spearman")
-    >>> print(f"Removed correlated columns")
+    >>> new_df = correlation_reduction(df=my_df, dv='target', threshold=0.85, method="spearman")
+    >>> new_df = correlation_reduction(df=my_df, dv='target', min_periods=100)
     """
     # Parameter validation
     if not isinstance(df, pd.DataFrame):
@@ -354,6 +440,15 @@ def correlation_reduction(
     if method not in valid_methods:
         raise ValueError(f"method must be one of {valid_methods}, got {method}")
     
+    if not isinstance(dropna, bool):
+        raise TypeError(f"dropna must be a bool, got {type(dropna)}")
+    
+    if isinstance(min_periods, bool) or not isinstance(min_periods, int):
+        raise TypeError(f"min_periods must be an int, got {type(min_periods)}")
+    
+    if min_periods < 1:
+        raise ValueError(f"min_periods must be at least 1, got {min_periods}")
+    
     # Create a deep copy of the dataframe
     work_df = df.copy(deep=True)
     
@@ -364,46 +459,47 @@ def correlation_reduction(
         # Remove the target variable from the list of features to consider for removal
         feature_cols.remove(dv)
     
-    # Ensure we have the target in numeric_cols for correlation calculations if it's numeric
-    numeric_cols = feature_cols.copy()
-    if dv is not None and dv in work_df.columns and pd.api.types.is_numeric_dtype(work_df[dv]):
-        numeric_cols.append(dv)
-    
     # Check if we have enough numeric columns to proceed
     if len(feature_cols) < 2:
         if verbose:
             print("Not enough numeric columns for correlation analysis")
         return work_df
     
+    if method == "mutual_info" and not dropna:
+        check_cols = feature_cols + ([dv] if dv is not None else [])
+        if work_df[check_cols].isna().any().any():
+            raise ValueError(
+                "mutual_info cannot be computed on missing values; set dropna=True to use "
+                "the rows where both columns of each pair are non-null"
+            )
+    
     # Calculate correlations based on the chosen method
     if method in ["pearson", "spearman"]:
-        # Standard correlation methods
-        correlation_matrix = work_df[numeric_cols].corr(method=method).abs()
-        
-        # Create a list of feature pairs and their correlation
-        corr_pairs = []
-        for i in range(len(feature_cols)):
-            for j in range(i+1, len(feature_cols)):
-                # Get the column names
-                col1 = feature_cols[i]
-                col2 = feature_cols[j]
-                
-                # Find their positions in the correlation matrix
-                idx1 = numeric_cols.index(col1)
-                idx2 = numeric_cols.index(col2)
-                
-                # Check if correlation exceeds threshold
-                if correlation_matrix.iloc[idx1, idx2] > threshold:
-                    corr_pairs.append({
-                        "feature1": col1,
-                        "feature2": col2,
-                        "correlation": correlation_matrix.iloc[idx1, idx2]
-                    })
+        # Standard correlation methods; pairs above the threshold from the upper triangle
+        correlation_matrix = work_df[feature_cols].corr(method=method, min_periods=min_periods).abs().to_numpy()
+        rows, cols = np.triu_indices(len(feature_cols), k=1)
+        pair_corr = correlation_matrix[rows, cols]
+        above = pair_corr > threshold
+        corr_pairs = [
+            {"feature1": feature_cols[i], "feature2": feature_cols[j], "correlation": corr}
+            for i, j, corr in zip(rows[above], cols[above], pair_corr[above])
+        ]
     
     elif method == "mutual_info":
         # For mutual information between features
         corr_pairs = []
         X = work_df[feature_cols]
+        non_null = X.notna()
+        complete = non_null.all()
+        n_unique = X.nunique()
+        min_rows = max(min_periods, 4)
+        
+        # MI of each feature with itself, used for normalization; computed once for
+        # features without nulls, per pair otherwise
+        full_entropy = {}
+        
+        def self_mi(values: pd.Series) -> float:
+            return mutual_info_regression(values.to_frame(), values.to_numpy())[0]
         
         # Calculate pairwise mutual information
         for i in range(len(feature_cols)):
@@ -412,23 +508,29 @@ def correlation_reduction(
                 feature2 = feature_cols[j]
                 
                 # Skip if either column has all identical values
-                if X[feature1].nunique() <= 1 or X[feature2].nunique() <= 1:
+                if n_unique[feature1] <= 1 or n_unique[feature2] <= 1:
                     continue
                 
-                # Calculate mutual information between the two features
-                mi = mutual_info_regression(
-                    X[[feature1]], X[feature2].values.reshape(-1, 1)
-                )[0]
+                if complete[feature1] and complete[feature2]:
+                    if len(X) < min_rows:
+                        continue
+                    x1, x2 = X[feature1], X[feature2]
+                    if feature1 not in full_entropy:
+                        full_entropy[feature1] = self_mi(x1)
+                    if feature2 not in full_entropy:
+                        full_entropy[feature2] = self_mi(x2)
+                    entropy1, entropy2 = full_entropy[feature1], full_entropy[feature2]
+                else:
+                    pair_rows = non_null[feature1] & non_null[feature2]
+                    if pair_rows.sum() < min_rows:
+                        continue
+                    x1, x2 = X.loc[pair_rows, feature1], X.loc[pair_rows, feature2]
+                    entropy1, entropy2 = self_mi(x1), self_mi(x2)
                 
-                # Normalize by entropy for a 0-1 scale
-                entropy1 = mutual_info_regression(
-                    X[[feature1]], X[feature1].values.reshape(-1, 1)
-                )[0]
-                entropy2 = mutual_info_regression(
-                    X[[feature2]], X[feature2].values.reshape(-1, 1)
-                )[0]
+                # Calculate mutual information between the two features
+                mi = mutual_info_regression(x1.to_frame(), x2.to_numpy())[0]
                 
-                # Avoid division by zero
+                # Normalize by entropy for a 0-1 scale; avoid division by zero
                 if entropy1 > 0 and entropy2 > 0:
                     normalized_mi = mi / min(entropy1, entropy2)
                     
@@ -448,21 +550,17 @@ def correlation_reduction(
     
     if dv is not None and dv in work_df.columns:
         # If we have a target variable, calculate correlations to the target
+        y = work_df[dv]
         if method in ["pearson", "spearman"]:
-            target_corr = work_df[numeric_cols].corr(method=method)[dv].abs()
-            target_corr = target_corr.drop(dv) if dv in target_corr.index else target_corr
+            target_corr = pd.Series(
+                {col: work_df[col].corr(y, method=method, min_periods=min_periods) for col in feature_cols},
+                dtype=float
+            ).abs()
         else:  # mutual_info
-            X = work_df[feature_cols]  # Only use feature columns, not target
-            y = work_df[dv]
-            
-            # Determine if classification or regression
-            if pd.api.types.is_categorical_dtype(y) or y.nunique() < 10:
-                mi_values = mutual_info_classif(X, y)
-            else:
-                mi_values = mutual_info_regression(X, y)
-            
-            # Create a Series for target correlations
-            target_corr = pd.Series(mi_values, index=feature_cols)
+            target_corr = _mutual_info_to_target(work_df[feature_cols], y, min_periods)
+        
+        # A feature whose target correlation cannot be computed ranks below any real one
+        target_corr = target_corr.fillna(0)
         
         # Process each correlated pair
         for pair in corr_pairs:
@@ -524,5 +622,3 @@ def correlation_reduction(
         work_df = work_df.drop(columns=dropped_columns)
     
     return work_df
-    
-    return work_df

{gitlabds-2.4.0 → gitlabds-2.4.1}/gitlabds.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gitlabds
-Version: 2.4.0
+Version: 2.4.1
 Summary: GitLab Data Science Tools
 Author-email: Kevin Dietz <kdietz@gitlab.com>
 License-Expression: MIT
@@ -376,9 +376,9 @@ test_df_transformed = gitlabds.apply_dummy(
 <details><summary> Remove Low Variation </summary>
 
 #### Description
-Remove columns from a dataset that do not meet the variation threshold. That is, columns will be dropped that contain a high percentage of one value.
+Remove columns from a dataset that do not meet the variation threshold. That is, columns will be dropped that contain a high percentage of one value. Columns that are entirely null are always dropped.
 
-`gitlabds.remove_low_variation(df=None, dv=None, columns='all', threshold=.98, verbose=True):`
+`gitlabds.remove_low_variation(df=None, dv=None, columns='all', threshold=.98, verbose=True, dropna=False):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -386,6 +386,7 @@ Remove columns from a dataset that do not meet the variation threshold. That is,
 - **_columns_** : Will examine at all columns by default. To limit to just a subset of columns, pass a list of column names. 
 - **_threshold_**: The maximum percentage one value in a column can represent. columns that exceed this threshold will be dropped. For example, the default value of `0.98` will drop any column where one value is present in more than 98% of rows.
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `False` (default) to count missing values as their own value. Set to `True` to measure variation over non-null values only, so a column that is one value plus NaN is dropped. Use `True` when missing values are left in place for a model that handles them natively (e.g. XGBoost, LightGBM).
 
 #### Returns
 - DataFrame with low variation columns dropped.
@@ -399,6 +400,10 @@ new_df = gitlabds.remove_low_variation(df=my_df, dv='my_outcome', columns='all',
 # Drop any of the selected columns where one value is present in more than 99% of rows.
 new_df = gitlabds.remove_low_variation(df=my_df, dv=None, columns=['colA', 'colB', 'colC'], threshold=.99)
 ```
+```python
+# Ignore missing values: drop columns where one value makes up more than 98% of the non-null rows.
+new_df = gitlabds.remove_low_variation(df=my_df, dv='my_outcome', dropna=True)
+```
 </details>
 
 <details><summary> Correlation Reduction </summary>
@@ -406,7 +411,7 @@ new_df = gitlabds.remove_low_variation(df=my_df, dv=None, columns=['colA', 'colB
 #### Description
 Reduce the number of columns on a dataframe by dropping columns that are highly correlated with other columns. Note: only one of the two highly correlated columns will be dropped.
 
-`gitlabds.correlation_reduction(df=None, dv=None, threshold=0.9, method="pearson", verbose=True):`
+`gitlabds.correlation_reduction(df=None, dv=None, threshold=0.9, method="pearson", verbose=True, dropna=True, min_periods=1):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -414,6 +419,8 @@ Reduce the number of columns on a dataframe by dropping columns that are highly
 - **_threshold_**: The threshold above which columns will be dropped. If two variables exceed this threshold, one will be dropped from the dataframe. For example, the default value of `0.90` will identify columns that have correlations greater than 90% to each other and drop one of those columns.
 - **_method_**: The correlation method to use. Options are "pearson" (linear relationships), "spearman" (monotonic relationships), or "mutual_info" (any statistical dependency).
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `True` (default) to compare each pair of columns (and each column with the outcome) using only rows where both are non-null. Pearson and Spearman always work this way; for "mutual_info", `False` raises an error if any missing values are present.
+- **_min_periods_** : Minimum number of rows where both columns of a pair are non-null for the pair to be evaluated. Pairs below it are never treated as correlated. Guards against spurious high correlations between sparsely populated columns. "mutual_info" always requires at least 4 such rows.
 
 #### Returns
 - DataFrame with redundant correlated columns dropped.
@@ -434,7 +441,7 @@ new_df = gitlabds.correlation_reduction(df=my_df, dv=None, threshold=0.90, metho
 #### Description
 Remove columns that are highly correlated with the outcome (target) column.
 
-`gitlabds.remove_outcome_proxies(df, dv, threshold=.8, method="pearson", verbose=True):`
+`gitlabds.remove_outcome_proxies(df, dv, threshold=.8, method="pearson", verbose=True, dropna=True, min_periods=1):`
 
 #### Parameters:
 - _**df**_ : your pandas dataframe
@@ -442,6 +449,8 @@ Remove columns that are highly correlated with the outcome (target) column.
 - _**threshold**_ : The correlation value to the outcome above which columns will be dropped. For example, the default value of `0.80` will identify and drop columns that have correlations greater than 80% to the outcome.
 - **_method_**: The correlation method to use. Options are "pearson" (linear relationships), "spearman" (monotonic relationships), or "mutual_info" (any statistical dependency).
 - **_verbose_** : Set to `True` to print outputs of columns being dropped. Set to `False` to suppress.
+- **_dropna_** : Set to `True` (default) to compare each column to the outcome using only rows where both are non-null. Pearson and Spearman always work this way; for "mutual_info", `False` raises an error if any missing values are present.
+- **_min_periods_** : Minimum number of rows where both the column and the outcome are non-null for the column to be evaluated. Columns below it are never dropped. Guards against spurious high correlations from sparsely populated columns. "mutual_info" always requires at least 4 such rows.
 
 #### Returns
 - DataFrame with outcome proxy columns dropped.
@@ -455,6 +464,10 @@ new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', threshold=.7
 # Drop columns with correlations to the outcome greater than 80% using Spearman correlation
 new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', threshold=.8, method="spearman")        
 ```
+```python
+# Data with missing values: only evaluate columns with at least 100 non-null rows alongside the outcome
+new_df = gitlabds.remove_outcome_proxies(df=my_df, dv='my_outcome', method="spearman", min_periods=100)
+```
 </details>
 
 <details><summary> Drop Categorical </summary>

{gitlabds-2.4.0 → gitlabds-2.4.1}/gitlabds.egg-info/SOURCES.txt

@@ -24,4 +24,5 @@ gitlabds.egg-info/SOURCES.txt
 gitlabds.egg-info/dependency_links.txt
 gitlabds.egg-info/requires.txt
 gitlabds.egg-info/top_level.txt
+tests/test_feature_reduction.py
 tests/test_snowflake_connectors.py

gitlabds-2.4.1/tests/test_feature_reduction.py

@@ -0,0 +1,232 @@
+"""
+Unit tests for gitlabds.feature_reduction: remove_low_variation, remove_outcome_proxies,
+correlation_reduction.
+
+Run with:  pytest tests/test_feature_reduction.py
+"""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from gitlabds import correlation_reduction, remove_low_variation, remove_outcome_proxies
+
+
+# ---------------------------------------------------------------------------
+# remove_low_variation
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def low_variation_df():
+    n = 100
+    rng = np.random.default_rng(0)
+    return pd.DataFrame({
+        "target": rng.integers(0, 2, n),
+        "varied": rng.normal(size=n),
+        "one_value_plus_nan": [1.0] * 83 + [np.nan] * 17,
+        "two_values_plus_nan": [1.0] * 50 + [2.0] * 30 + [np.nan] * 20,
+        "mostly_nan": [1.0] + [np.nan] * 99,
+        "all_null": [np.nan] * n,
+        "constant": [5] * n,
+    })
+
+
+class TestRemoveLowVariation:
+    def test_default_counts_nan_as_value(self, low_variation_df):
+        out = remove_low_variation(low_variation_df, dv="target", verbose=False)
+        assert set(out.columns) == {"target", "varied", "one_value_plus_nan", "two_values_plus_nan"}
+
+    def test_dropna_uses_non_null_values_only(self, low_variation_df):
+        out = remove_low_variation(low_variation_df, dv="target", dropna=True, verbose=False)
+        assert set(out.columns) == {"target", "varied", "two_values_plus_nan"}
+
+    @pytest.mark.parametrize("dropna", [False, True])
+    def test_all_null_column_dropped(self, low_variation_df, dropna):
+        out = remove_low_variation(low_variation_df, dropna=dropna, verbose=False)
+        assert "all_null" not in out.columns
+
+    def test_dropna_threshold_on_non_null_share(self):
+        # 97 of 98 non-null rows are 0 (99.0%); counting NaN, 0 is only 97%
+        df = pd.DataFrame({"x": [0.0] * 97 + [1.0] + [np.nan] * 2})
+        assert "x" in remove_low_variation(df, threshold=0.98, verbose=False).columns
+        assert "x" not in remove_low_variation(df, threshold=0.98, dropna=True, verbose=False).columns
+
+    def test_does_not_mutate_columns_argument(self, low_variation_df):
+        cols = ["target", "constant"]
+        remove_low_variation(low_variation_df, dv="target", columns=cols, verbose=False)
+        assert cols == ["target", "constant"]
+
+    def test_invalid_dropna(self, low_variation_df):
+        with pytest.raises(TypeError):
+            remove_low_variation(low_variation_df, dropna="yes")
+
+
+# ---------------------------------------------------------------------------
+# remove_outcome_proxies
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def proxy_df():
+    n = 500
+    rng = np.random.default_rng(1)
+    y = rng.normal(size=n)
+    df = pd.DataFrame({
+        "y": y,
+        "proxy": y + rng.normal(scale=0.05, size=n),
+        "noise": rng.normal(size=n),
+        "weak": 0.3 * y + rng.normal(size=n),
+    })
+    df["proxy_with_nan"] = df["proxy"].where(rng.random(n) > 0.3)
+    df["noise_with_nan"] = df["noise"].where(rng.random(n) > 0.3)
+    df["sparse"] = np.nan
+    df.loc[:2, "sparse"] = [0.0, 1.0, 2.0]
+    df.loc[:2, "y"] = [0.0, 1.0, 2.1]
+    return df
+
+
+class TestRemoveOutcomeProxies:
+    @pytest.mark.parametrize("method", ["pearson", "spearman"])
+    def test_matches_full_correlation_matrix(self, proxy_df, method):
+        expected = proxy_df.corr(method=method)["y"].drop("y").abs()
+        expected = set(expected[expected > 0.3].index)
+        out = remove_outcome_proxies(proxy_df, dv="y", threshold=0.3, method=method, verbose=False)
+        assert set(proxy_df.columns) - set(out.columns) == expected
+
+    @pytest.mark.parametrize("method", ["pearson", "spearman"])
+    def test_min_periods_guards_sparse_columns(self, proxy_df, method):
+        # "sparse" has 3 non-null rows that line up perfectly with y
+        out = remove_outcome_proxies(proxy_df, dv="y", method=method, verbose=False)
+        assert "sparse" not in out.columns
+        out = remove_outcome_proxies(proxy_df, dv="y", method=method, min_periods=30, verbose=False)
+        assert "sparse" in out.columns
+        assert "proxy" not in out.columns
+
+    def test_mutual_info_handles_nan(self, proxy_df):
+        out = remove_outcome_proxies(proxy_df, dv="y", method="mutual_info", threshold=0.5, verbose=False)
+        assert "proxy" not in out.columns
+        assert "proxy_with_nan" not in out.columns
+        assert {"noise", "noise_with_nan", "weak", "sparse"} <= set(out.columns)
+
+    def test_mutual_info_handles_nan_in_dv(self, proxy_df):
+        proxy_df.loc[proxy_df.index[-50:], "y"] = np.nan
+        out = remove_outcome_proxies(proxy_df, dv="y", method="mutual_info", threshold=0.5, verbose=False)
+        assert "proxy" not in out.columns
+        assert "noise" in out.columns
+
+    def test_mutual_info_dropna_false_raises_on_nan(self, proxy_df):
+        with pytest.raises(ValueError, match="dropna=True"):
+            remove_outcome_proxies(proxy_df, dv="y", method="mutual_info", dropna=False)
+
+    def test_mutual_info_dropna_false_without_nan(self, proxy_df):
+        df = proxy_df[["y", "proxy", "noise"]]
+        out = remove_outcome_proxies(df, dv="y", method="mutual_info", threshold=0.5, dropna=False, verbose=False)
+        assert list(out.columns) == ["y", "noise"]
+
+    def test_mutual_info_classification_dv(self, proxy_df):
+        df = proxy_df.assign(y=(proxy_df["y"] > 0).astype(int), proxy=(proxy_df["y"] > 0).astype(float))
+        df.loc[df.index[:100], "proxy"] = np.nan
+        out = remove_outcome_proxies(df, dv="y", method="mutual_info", threshold=0.5, verbose=False)
+        assert "proxy" not in out.columns
+        assert "noise" in out.columns
+
+    @pytest.mark.parametrize("min_periods", [0, 1.5, True])
+    def test_invalid_min_periods(self, proxy_df, min_periods):
+        with pytest.raises((TypeError, ValueError)):
+            remove_outcome_proxies(proxy_df, dv="y", min_periods=min_periods)
+
+
+# ---------------------------------------------------------------------------
+# correlation_reduction
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def redundant_df():
+    n = 400
+    rng = np.random.default_rng(2)
+    a = rng.normal(size=n)
+    b = rng.normal(size=n)
+    df = pd.DataFrame({
+        "y": a + rng.normal(scale=0.5, size=n),
+        "a1": a + rng.normal(scale=0.1, size=n),
+        "a2": a + rng.normal(scale=0.2, size=n),
+        "b1": b,
+        "b2": b + rng.normal(scale=0.1, size=n),
+        "noise": rng.normal(size=n),
+    })
+    df["a2"] = df["a2"].where(rng.random(n) > 0.2)
+    df["b2"] = df["b2"].where(rng.random(n) > 0.2)
+    return df
+
+
+def _reference_drops(df, dv, threshold, method):
+    """The pre-vectorization pair search, for pearson/spearman."""
+    features = [c for c in df.select_dtypes("number").columns if c != dv]
+    corr = df[features].corr(method=method).abs()
+    pairs = [
+        (corr.iloc[i, j], features[i], features[j])
+        for i in range(len(features)) for j in range(i + 1, len(features))
+        if corr.iloc[i, j] > threshold
+    ]
+    pairs.sort(key=lambda p: p[0], reverse=True)
+    target = df[features].corrwith(df[dv], method=method).abs() if dv else None
+    dropped = set()
+    for _, f1, f2 in pairs:
+        if f1 in dropped or f2 in dropped:
+            continue
+        if target is None or target[f1] >= target[f2]:
+            dropped.add(f2)
+        else:
+            dropped.add(f1)
+    return dropped
+
+
+class TestCorrelationReduction:
+    @pytest.mark.parametrize("method", ["pearson", "spearman"])
+    @pytest.mark.parametrize("dv", [None, "y"])
+    def test_matches_reference(self, redundant_df, method, dv):
+        out = correlation_reduction(redundant_df, dv=dv, threshold=0.8, method=method, verbose=False)
+        dropped = set(redundant_df.columns) - set(out.columns)
+        assert dropped == _reference_drops(redundant_df, dv, 0.8, method)
+        assert {"a2", "b2"} <= dropped
+
+    def test_keeps_feature_with_target_correlation_over_nan(self):
+        n = 50
+        x = np.arange(n, dtype=float)
+        df = pd.DataFrame({"y": x, "f1": x, "f2": x})
+        # f2's target correlation is NaN (dv constant where f2 overlaps it); f1's is real
+        df.loc[: n // 2, "y"] = 1.0
+        df.loc[n // 2 + 1:, "f2"] = np.nan
+        out = correlation_reduction(df, dv="y", threshold=0.8, verbose=False)
+        assert list(out.columns) == ["y", "f1"]
+
+    def test_min_periods_guards_sparse_pairs(self):
+        n = 200
+        rng = np.random.default_rng(3)
+        df = pd.DataFrame({"s1": np.nan, "s2": np.nan, "x": rng.normal(size=n)}, index=range(n))
+        df.loc[:2, "s1"] = [0.0, 1.0, 2.0]
+        df.loc[:2, "s2"] = [0.0, 1.0, 2.1]
+        assert "s2" not in correlation_reduction(df, verbose=False).columns
+        assert list(correlation_reduction(df, min_periods=30, verbose=False).columns) == ["s1", "s2", "x"]
+
+    @pytest.mark.parametrize("dv", [None, "y"])
+    def test_mutual_info_handles_nan(self, redundant_df, dv):
+        out = correlation_reduction(redundant_df, dv=dv, threshold=0.5, method="mutual_info", verbose=False)
+        assert {"noise", "y"} <= set(out.columns)
+        assert not {"b1", "b2"} <= set(out.columns)
+
+    def test_mutual_info_dropna_false_raises_on_nan(self, redundant_df):
+        with pytest.raises(ValueError, match="dropna=True"):
+            correlation_reduction(redundant_df, method="mutual_info", dropna=False)
+
+    def test_mutual_info_dropna_false_without_nan(self, redundant_df):
+        df = redundant_df.drop(columns=["a2", "b2"])
+        out = correlation_reduction(df, dv="y", threshold=0.5, method="mutual_info", dropna=False, verbose=False)
+        assert "noise" in out.columns
+
+    @pytest.mark.parametrize("min_periods", [0, 1.5, True])
+    def test_invalid_min_periods(self, redundant_df, min_periods):
+        with pytest.raises((TypeError, ValueError)):
+            correlation_reduction(redundant_df, min_periods=min_periods)