dragon-ml-toolbox 1.3.2__py3-none-any.whl → 1.4.1__py3-none-any.whl

ml_tools/data_exploration.py

@@ -2,34 +2,30 @@ import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
 import seaborn as sns
-from statsmodels.stats.outliers_influence import variance_inflation_factor
-from statsmodels.tools.tools import add_constant
 from IPython import get_ipython
 from IPython.display import clear_output
 import time
-from typing import Union, Literal, Dict, Tuple, Optional
+from typing import Union, Literal, Dict, Tuple
 import os
 import sys
 import textwrap
-from ml_tools.utilities import sanitize_filename
-
-
-# Keep track of all available functions, show using `info()`
-__all__ = ["summarize_dataframe",
-           "drop_rows_with_missing_data",
-           "split_features_targets", 
-           "show_null_columns",
-           "drop_columns_with_missing_data",
-           "split_continuous_binary",
-           "plot_correlation_heatmap",
-           "check_value_distributions",
-           "plot_value_distributions",
-           "clip_outliers_single",
-           "clip_outliers_multi",
-           "merge_dataframes",
-           "save_dataframe",
-           "compute_vif",
-           "drop_vif_based"]
+from ml_tools.utilities import sanitize_filename, _script_info
+
+
+# Keep track of all available tools, show using `info()`
+__all__ = [
+    "summarize_dataframe",
+    "drop_rows_with_missing_data",
+    "split_features_targets", 
+    "show_null_columns",
+    "drop_columns_with_missing_data",
+    "split_continuous_binary", 
+    "plot_correlation_heatmap", 
+    "check_value_distributions", 
+    "plot_value_distributions", 
+    "clip_outliers_single", 
+    "clip_outliers_multi"
+]
 
 
 def summarize_dataframe(df: pd.DataFrame, round_digits: int = 2):
@@ -63,34 +59,6 @@ def summarize_dataframe(df: pd.DataFrame, round_digits: int = 2):
     return summary
 
 
-def show_null_columns(df: pd.DataFrame, round_digits: int = 2):
-    """
-    Displays a table of columns with missing values, showing both the count and
-    percentage of missing entries per column.
-
-    Parameters:
-        df (pd.DataFrame): The input DataFrame.
-        round_digits (int): Number of decimal places for the percentage.
-        
-    Returns:
-        pd.DataFrame: A DataFrame summarizing missing values in each column.
-    """
-    null_counts = df.isnull().sum()
-    null_percent = df.isnull().mean() * 100
-
-    # Filter only columns with at least one null
-    mask = null_counts > 0
-    null_summary = pd.DataFrame({
-        'Missing Count': null_counts[mask],
-        'Missing %': null_percent[mask].round(round_digits)
-    })
-
-    # Sort by descending percentage of missing values
-    null_summary = null_summary.sort_values(by='Missing %', ascending=False)
-    # print(null_summary)
-    return null_summary
-
-
 def drop_rows_with_missing_data(df: pd.DataFrame, threshold: float = 0.7) -> pd.DataFrame:
     """
     Drops rows with more than `threshold` fraction of missing values.
@@ -137,6 +105,57 @@ def split_features_targets(df: pd.DataFrame, targets: list[str]):
     return df_targets, df_features
 
 
+def show_null_columns(df: pd.DataFrame, round_digits: int = 2):
+    """
+    Displays a table of columns with missing values, showing both the count and
+    percentage of missing entries per column.
+
+    Parameters:
+        df (pd.DataFrame): The input DataFrame.
+        round_digits (int): Number of decimal places for the percentage.
+        
+    Returns:
+        pd.DataFrame: A DataFrame summarizing missing values in each column.
+    """
+    null_counts = df.isnull().sum()
+    null_percent = df.isnull().mean() * 100
+
+    # Filter only columns with at least one null
+    mask = null_counts > 0
+    null_summary = pd.DataFrame({
+        'Missing Count': null_counts[mask],
+        'Missing %': null_percent[mask].round(round_digits)
+    })
+
+    # Sort by descending percentage of missing values
+    null_summary = null_summary.sort_values(by='Missing %', ascending=False)
+    # print(null_summary)
+    return null_summary
+
+
+def drop_columns_with_missing_data(df: pd.DataFrame, threshold: float = 0.7) -> pd.DataFrame:
+    """
+    Drops columns with more than `threshold` fraction of missing values.
+
+    Parameters:
+        df (pd.DataFrame): The input DataFrame.
+        threshold (float): Fraction of missing values above which columns are dropped.
+
+    Returns:
+        pd.DataFrame: A new DataFrame without the dropped columns.
+    """
+    missing_fraction = df.isnull().mean()
+    cols_to_drop = missing_fraction[missing_fraction > threshold].index
+
+    if len(cols_to_drop) > 0:
+        print(f"Dropping columns with more than {threshold*100:.0f}% missing data:")
+        print(list(cols_to_drop))
+    else:
+        print(f"No columns have more than {threshold*100:.0f}% missing data.")
+
+    return df.drop(columns=cols_to_drop)
+
+
 def split_continuous_binary(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
     """
     Split DataFrame into two DataFrames: one with continuous columns, one with binary columns.
@@ -179,29 +198,6 @@ def split_continuous_binary(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFram
 
     return df_cont, df_bin # type: ignore
 
-    
-def drop_columns_with_missing_data(df: pd.DataFrame, threshold: float = 0.7) -> pd.DataFrame:
-    """
-    Drops columns with more than `threshold` fraction of missing values.
-
-    Parameters:
-        df (pd.DataFrame): The input DataFrame.
-        threshold (float): Fraction of missing values above which columns are dropped.
-
-    Returns:
-        pd.DataFrame: A new DataFrame without the dropped columns.
-    """
-    missing_fraction = df.isnull().mean()
-    cols_to_drop = missing_fraction[missing_fraction > threshold].index
-
-    if len(cols_to_drop) > 0:
-        print(f"Dropping columns with more than {threshold*100:.0f}% missing data:")
-        print(list(cols_to_drop))
-    else:
-        print(f"No columns have more than {threshold*100:.0f}% missing data.")
-
-    return df.drop(columns=cols_to_drop)
-
 
 def plot_correlation_heatmap(df: pd.DataFrame, save_dir: Union[str, None] = None, method: Literal["pearson", "kendall", "spearman"]="pearson", plot_title: str="Correlation Heatmap"):
     """
@@ -278,7 +274,7 @@ def check_value_distributions(df: pd.DataFrame, view_frequencies: bool=True, bin
     Notes:
         - Binning is adaptive: if quantile binning results in ≤ 2 unique bins, raw values are used instead.
     """
-    # cherrypick columns
+    # cherry-pick columns
     if skip_cols_with_key is not None:
         columns = [col for col in df.columns if skip_cols_with_key not in col]
     else:
@@ -351,7 +347,7 @@ def plot_value_distributions(df: pd.DataFrame, save_dir: str, bin_threshold: int
     dict_to_plot_std = dict()
     dict_to_plot_freq = dict()
     
-    # cherrypick columns
+    # cherry-pick columns
     if skip_cols_with_key is not None:
         columns = [col for col in df.columns if skip_cols_with_key not in col]
     else:
@@ -399,7 +395,7 @@ def plot_value_distributions(df: pd.DataFrame, save_dir: str, bin_threshold: int
                 labels = data.keys()
                 
             plt.figure(figsize=(10, 6))
-            colors = plt.cm.tab20.colors if len(data) <= 20 else plt.cm.viridis(np.linspace(0, 1, len(data)))
+            colors = plt.cm.tab20.colors if len(data) <= 20 else plt.cm.viridis(np.linspace(0, 1, len(data))) # type: ignore
                 
             plt.bar(labels, data.values(), color=colors[:len(data)], alpha=0.85)
             plt.xlabel("Values", fontsize=base_fontsize)
@@ -518,218 +514,10 @@ def clip_outliers_multi(
     return new_df
 
 
-def merge_dataframes(
-    *dfs: pd.DataFrame,
-    reset_index: bool = False,
-    direction: Literal["horizontal", "vertical"] = "horizontal"
-) -> pd.DataFrame:
-    """
-    Merges multiple DataFrames either horizontally or vertically.
-
-    Parameters:
-        *dfs (pd.DataFrame): Variable number of DataFrames to merge.
-        reset_index (bool): Whether to reset index in the final merged DataFrame.
-        direction (["horizontal" | "vertical"]):
-            - "horizontal": Merge on index, adding columns.
-            - "vertical": Append rows; all DataFrames must have identical columns.
-
-    Returns:
-        pd.DataFrame: A single merged DataFrame.
-
-    Raises:
-        ValueError:
-            - If fewer than 2 DataFrames are provided.
-            - If indexes do not match for horizontal merge.
-            - If column names or order differ for vertical merge.
-    """
-    if len(dfs) < 2:
-        raise ValueError("At least 2 DataFrames must be provided.")
-    
-    for i, df in enumerate(dfs, start=1):
-        print(f"DataFrame {i} shape: {df.shape}")
-    
-
-    if direction == "horizontal":
-        reference_index = dfs[0].index
-        for i, df in enumerate(dfs, start=1):
-            if not df.index.equals(reference_index):
-                raise ValueError(f"Indexes do not match: Dataset 1 and Dataset {i}.")
-        merged_df = pd.concat(dfs, axis=1)
-
-    elif direction == "vertical":
-        reference_columns = dfs[0].columns
-        for i, df in enumerate(dfs, start=1):
-            if not df.columns.equals(reference_columns):
-                raise ValueError(f"Column names/order do not match: Dataset 1 and Dataset {i}.")
-        merged_df = pd.concat(dfs, axis=0)
-
-    else:
-        raise ValueError(f"Invalid merge direction: {direction}")
-
-    if reset_index:
-        merged_df = merged_df.reset_index(drop=True)
-
-    print(f"Merged DataFrame shape: {merged_df.shape}")
-
-    return merged_df
-
-
-def save_dataframe(df: pd.DataFrame, save_dir: str, filename: str) -> None:
-    """
-    Save a pandas DataFrame to a CSV file.
-
-    Parameters:
-        df: pandas.DataFrame to save
-        save_dir: str, directory where the CSV file will be saved.
-        filename: str, CSV filename, extension will be added if missing.
-    """
-    os.makedirs(save_dir, exist_ok=True)
-    
-    filename = sanitize_filename(filename)
-    
-    if not filename.endswith('.csv'):
-        filename += '.csv'
-        
-    output_path = os.path.join(save_dir, filename)
-        
-    df.to_csv(output_path, index=False, encoding='utf-8')
-    print(f"Saved file: '{filename}'")
-
-
-def compute_vif(
-    df: pd.DataFrame,
-    features: Optional[list[str]] = None,
-    ignore_cols: Optional[list[str]] = None,
-    plot: bool = True,
-    save_dir: Union[str, None] = None
-) -> pd.DataFrame:
-    """
-    Computes Variance Inflation Factors (VIF) for numeric features, optionally plots and saves the results.
-    
-    There cannot be empty values in the dataset.
-
-    Args:
-        df (pd.DataFrame): The input DataFrame.
-        features (list[str] | None): Optional list of column names to evaluate. Defaults to all numeric columns.
-        ignore_cols (list[str] | None): Optional list of column names to ignore.
-        plot (bool): Whether to display a barplot of VIF values.
-        save_dir (str | None): Directory to save the plot as SVG. If None, plot is not saved.
-
-    Returns:
-        pd.DataFrame: DataFrame with features and corresponding VIF values, sorted descending.
-    
-    NOTE:
-    **Variance Inflation Factor (VIF)** quantifies the degree of multicollinearity among features in a dataset. 
-    A VIF value indicates how much the variance of a regression coefficient is inflated due to linear dependence with other features. 
-    A VIF of 1 suggests no correlation, values between 1 and 5 indicate moderate correlation, and values greater than 10 typically signal high multicollinearity, which may distort model interpretation and degrade performance.
-        
-    """
-    if features is None:
-        features = df.select_dtypes(include='number').columns.tolist()
-    
-    if ignore_cols is not None:
-        missing = set(ignore_cols) - set(features)
-        if missing:
-            raise ValueError(f"The following 'columns to ignore' are not in the Dataframe:\n{missing}")
-        features = [f for f in features if f not in ignore_cols]
-
-    X = df[features].copy()
-    X = add_constant(X, has_constant='add')
-
-    vif_data = pd.DataFrame()
-    vif_data["feature"] = X.columns
-    vif_data["VIF"] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]
-
-    # Drop the constant column
-    vif_data = vif_data[vif_data["feature"] != "const"]
-    vif_data = vif_data.sort_values(by="VIF", ascending=False).reset_index(drop=True) # type: ignore
-
-    # Add color coding based on thresholds
-    def vif_color(v: float) -> str:
-        if v > 10:
-            return "red"
-        elif v > 5:
-            return "gold"
-        else:
-            return "green"
-
-    vif_data["color"] = vif_data["VIF"].apply(vif_color)
-
-    # Plot
-    if plot or save_dir:
-        plt.figure(figsize=(10, 6))
-        bars = plt.barh(
-            vif_data["feature"], 
-            vif_data["VIF"], 
-            color=vif_data["color"], 
-            edgecolor='black'
-        )
-        plt.title("Variance Inflation Factor (VIF) per Feature")
-        plt.xlabel("VIF")
-        plt.axvline(x=5, color='gold', linestyle='--', label='VIF = 5')
-        plt.axvline(x=10, color='red', linestyle='--', label='VIF = 10')
-        plt.legend(loc='lower right')
-        plt.gca().invert_yaxis()
-        plt.grid(axis='x', linestyle='--', alpha=0.5)
-        plt.tight_layout()
-
-        if save_dir:
-            os.makedirs(save_dir, exist_ok=True)
-            save_path = os.path.join(save_dir, "VIF_plot.svg")
-            plt.savefig(save_path, format='svg', bbox_inches='tight')
-            print(f"Saved VIF plot to: {save_path}")
-
-        if plot:
-            plt.show()
-        plt.close()
-
-    return vif_data.drop(columns="color")
-
-
-def drop_vif_based(df: pd.DataFrame, vif_df: pd.DataFrame, threshold: float = 10.0) -> pd.DataFrame:
-    """
-    Drops features from the original DataFrame based on their VIF values exceeding a given threshold.
-
-    Args:
-        df (pd.DataFrame): Original DataFrame containing the features.
-        vif_df (pd.DataFrame): DataFrame with 'feature' and 'VIF' columns as returned by `compute_vif()`.
-        threshold (float): VIF threshold above which features will be dropped.
-
-    Returns:
-        pd.DataFrame: A new DataFrame with high-VIF features removed.
-    """
-    # Ensure expected structure
-    if 'feature' not in vif_df.columns or 'VIF' not in vif_df.columns:
-        raise ValueError("`vif_df` must contain 'feature' and 'VIF' columns.")
-    
-    # Identify features to drop
-    to_drop = vif_df[vif_df["VIF"] > threshold]["feature"].tolist()
-    print(f"Dropping {len(to_drop)} feature(s) with VIF > {threshold}: {to_drop}")
-
-    return df.drop(columns=to_drop, errors="ignore")
-
-
 def _is_notebook():
     return get_ipython() is not None
 
 
-def info(full_info: bool=True):
-    """
-    List available functions and their descriptions.
-    """
-    print("Available functions for data exploration:")
-    if full_info:
-        module = sys.modules[__name__]
-        for name in __all__:
-            obj = getattr(module, name, None)
-            if callable(obj):
-                doc = obj.__doc__ or "No docstring provided."
-                formatted_doc = textwrap.indent(textwrap.dedent(doc.strip()), prefix="    ")
-                print(f"\n{name}:\n{formatted_doc}")
-    else:
-        for i, name in enumerate(__all__, start=1):
-            print(f"{i} - {name}")
-
+def info():
+    _script_info(__all__)
 
-if __name__ == "__main__":
-    info()

ml_tools/datasetmaster.py

@@ -11,6 +11,15 @@ from PIL import Image
 from torchvision.datasets import ImageFolder
 from torchvision import transforms
 import matplotlib.pyplot as plt
+from .utilities import _script_info
+
+
+__all__ = [
+    "DatasetMaker",
+    "PytorchDataset",
+    "make_vision_dataset",
+    "SequenceDataset",
+]
 
 
 class DatasetMaker():
@@ -592,4 +601,7 @@ class SequenceDataset():
         
     def __len__(self):
         return f"Train: {len(self.train_dataset)}, Test: {len(self.test_dataset)}"
-        
+
+
+def info():
+    _script_info(__all__)