dragon-ml-toolbox 1.4.0__py3-none-any.whl → 1.4.2__py3-none-any.whl

{dragon_ml_toolbox-1.4.0.dist-info → dragon_ml_toolbox-1.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 1.4.0
+Version: 1.4.2
 Summary: A collection of tools for data science and machine learning projects
 Author-email: Karl Loza <luigiloza@gmail.com>
 License-Expression: MIT
@@ -80,7 +80,7 @@ Clone the repository and install in editable mode with optional dependencies:
 ```bash
 git clone https://github.com/DrAg0n-BoRn/ML_tools.git
 cd ML_tools
-pip install -e '.[pytorch]'
+pip install -e .
 ```
 
 ## Usage
@@ -91,3 +91,19 @@ After installation, import modules like this:
 from ml_tools.utilities import sanitize_filename
 from ml_tools.logger import custom_logger
 ```
+
+## Available modules
+
+```bash
+data_exploration
+datasetmaster
+ensemble_learning
+handle_excel
+logger
+MICE_imputation
+particle_swarm_optimization
+trainer
+utilities
+VIF_factor
+vision_helpers
+```

dragon_ml_toolbox-1.4.2.dist-info/RECORD

@@ -0,0 +1,19 @@
+dragon_ml_toolbox-1.4.2.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
+dragon_ml_toolbox-1.4.2.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=e1Hg5ZtaBpDV7ZvxhLe1ac28l7nMjvi1MSE5YvB1s-o,1472
+ml_tools/MICE_imputation.py,sha256=CK0tYZ_kQkdETohOlhI7RP7oFkJTXrP-XtIxb--dzpU,9726
+ml_tools/VIF_factor.py,sha256=LQWr1P8WYij07FX_3RZC6Rr22bfAMnrt0Lhvi7SbBpY,9846
+ml_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ml_tools/data_exploration.py,sha256=FXP5i6bQo8J3RCyLRmlX-qJVh4VH8DbMjrdUmyd1mF0,18708
+ml_tools/datasetmaster.py,sha256=EFUEX-tqq94Ak1rXXYR-XaX85olrxvF2EuytdzUK7y0,29131
+ml_tools/ensemble_learning.py,sha256=p8t5PI63N3G0ZgvOKmvFOvwJ24qqPdZCvyiDAx4ggXY,27670
+ml_tools/handle_excel.py,sha256=ZJui5__0rc2T8UGHTheqZGhKmdVZ7Q2I54IoYCjAqJw,12612
+ml_tools/logger.py,sha256=NOtL3YSuffAGmpTpXjY-uJjqFLdRG_jpL7MDyloBw9c,4712
+ml_tools/particle_swarm_optimization.py,sha256=3xsc6sg-5o3cPbG_dWUyF3HdRVxgL4k_kRuPMU11NnM,20020
+ml_tools/pytorch_models.py,sha256=bpWZsrSwCvHJQkR6UfoPpElsMv9AvmiNErNHC8NYB_I,10132
+ml_tools/trainer.py,sha256=WAZ4EdrZuTOAnGXRWV3XcLNce4s7EKGf2-qchLC08Ik,15702
+ml_tools/utilities.py,sha256=Pou-8IZsZj9NiZ_shhLt552yaKNvbnQ1Ztoj6VMHIeE,10091
+ml_tools/vision_helpers.py,sha256=idQ-Ugp1IdsvwXiYyhYa9G3rTRTm37YRpkQDLEpANHM,7701
+dragon_ml_toolbox-1.4.2.dist-info/METADATA,sha256=c95w_AETVdAwMYWrowJKxkC0wYCsgRrTmxyekPz7WBE,2516
+dragon_ml_toolbox-1.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dragon_ml_toolbox-1.4.2.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
+dragon_ml_toolbox-1.4.2.dist-info/RECORD,,

ml_tools/MICE_imputation.py

@@ -3,9 +3,20 @@ import miceforest as mf
 import os
 import matplotlib.pyplot as plt
 import numpy as np
-from ml_tools.utilities import load_dataframe, list_csv_paths, sanitize_filename
+from ml_tools.utilities import load_dataframe, list_csv_paths, sanitize_filename, _script_info
 from plotnine import ggplot, labs, theme, element_blank # type: ignore
 
+
+__all__ = [
+    "apply_mice",
+    "save_imputed_datasets",
+    "get_na_column_names",
+    "get_convergence_diagnostic",
+    "get_imputed_distributions",
+    "run_mice_pipeline"
+]
+
+
 def apply_mice(df: pd.DataFrame, df_name: str, resulting_datasets: int=1, iterations: int=20, random_state: int=101):
     
     # Initialize kernel with number of imputed datasets to generate
@@ -210,7 +221,7 @@ def run_mice_pipeline(df_path_or_dir: str, save_datasets_dir: str, save_metrics_
     if os.path.isfile(df_path_or_dir):
         all_file_paths = [df_path_or_dir]
     elif os.path.isdir(df_path_or_dir):
-        all_file_paths = list_csv_paths(df_path_or_dir).values()
+        all_file_paths = list(list_csv_paths(df_path_or_dir).values())
     else:
         raise ValueError(f"Invalid path or directory: {df_path_or_dir}")
     
@@ -226,3 +237,7 @@ def run_mice_pipeline(df_path_or_dir: str, save_datasets_dir: str, save_metrics_
         get_convergence_diagnostic(kernel=kernel, imputed_dataset_names=imputed_dataset_names, column_names=imputed_column_names, root_dir=save_metrics_dir)
         
         get_imputed_distributions(kernel=kernel, df_name=df_name, root_dir=save_metrics_dir, column_names=imputed_column_names)
+
+
+def info():
+    _script_info(__all__)

ml_tools/VIF_factor.py

@@ -7,12 +7,19 @@ from statsmodels.stats.outliers_influence import variance_inflation_factor
 from statsmodels.tools.tools import add_constant
 import warnings
 import os
-from .utilities import sanitize_filename, yield_dataframes_from_dir, save_dataframe
+from .utilities import sanitize_filename, yield_dataframes_from_dir, save_dataframe, _script_info
+
+
+__all__ = [
+    "compute_vif",
+    "drop_vif_based",
+    "compute_vif_multi"
+]
 
 
 def compute_vif(
     df: pd.DataFrame,
-    target_columns: Optional[list[str]] = None,
+    use_columns: Optional[list[str]] = None,
     ignore_columns: Optional[list[str]] = None,
     max_features_to_plot: int = 20,
     save_dir: Optional[str] = None,
@@ -25,7 +32,7 @@ def compute_vif(
 
     Args:
         df (pd.DataFrame): The input DataFrame.
-        target_columns (list[str] | None): Optional list of columns to include. Defaults to all numeric columns.
+        use_columns (list[str] | None): Optional list of columns to include. Defaults to all numeric columns.
         ignore_columns (list[str] | None): Optional list of columns to exclude from the VIF computation. Skipped if `target_columns` is provided.
         max_features_to_plot (int): Adjust the number of features shown in the plot.
         save_dir (str | None): Directory to save the plot as SVG. If None, the plot is not saved.
@@ -42,20 +49,20 @@ def compute_vif(
     A VIF of 1 suggests no correlation, values between 1 and 5 indicate moderate correlation, and values greater than 10 typically signal high multicollinearity, which may distort model interpretation and degrade performance.
     """
     ground_truth_cols = df.columns.to_list()
-    if target_columns is None:
+    if use_columns is None:
         sanitized_columns = df.select_dtypes(include='number').columns.tolist()
         missing_features = set(ground_truth_cols) - set(sanitized_columns)
         if missing_features:
             print(f"⚠️ These columns are not Numeric:\n{missing_features}")
     else:
         sanitized_columns = list()
-        for feature in target_columns:
+        for feature in use_columns:
             if feature not in ground_truth_cols:
                 print(f"⚠️ The provided column '{feature}' is not in the DataFrame.")
             else:
                 sanitized_columns.append(feature)
     
-    if ignore_columns is not None and target_columns is None:
+    if ignore_columns is not None and use_columns is None:
         missing_ignore = set(ignore_columns) - set(ground_truth_cols)
         if missing_ignore:
             print(f"⚠️ Warning: The following 'columns to ignore' are not in the Dataframe:\n{missing_ignore}")
@@ -137,7 +144,7 @@ def compute_vif(
     return vif_data.drop(columns="color")
 
 
-def drop_vif_based(df: pd.DataFrame, vif_df: pd.DataFrame, threshold: float = 10.0) -> pd.DataFrame:
+def drop_vif_based(df: pd.DataFrame, vif_df: pd.DataFrame, threshold: float = 10.0) -> tuple[pd.DataFrame, list[str]]:
     """
     Drops columns from the original DataFrame based on their VIF values exceeding a given threshold.
 
@@ -147,7 +154,9 @@ def drop_vif_based(df: pd.DataFrame, vif_df: pd.DataFrame, threshold: float = 10
         threshold (float): VIF threshold above which columns will be dropped.
 
     Returns:
-        pd.DataFrame: A new DataFrame with high-VIF columns removed.
+        (tuple[pd.DataFrame, list[str]]): 
+            - A new DataFrame with high-VIF columns removed.
+            - A list with dropped column names.
     """
     # Ensure expected structure
     if 'feature' not in vif_df.columns or 'VIF' not in vif_df.columns:
@@ -162,13 +171,13 @@ def drop_vif_based(df: pd.DataFrame, vif_df: pd.DataFrame, threshold: float = 10
     if result_df.empty:
         print(f"\t⚠️ Warning: All columns were dropped.")
 
-    return result_df
+    return result_df, to_drop
 
 
 def compute_vif_multi(input_directory: str,
                       output_plot_directory: str,
                       output_dataset_directory: Optional[str] = None,
-                      target_columns: Optional[list[str]] = None,
+                      use_columns: Optional[list[str]] = None,
                       ignore_columns: Optional[list[str]] = None,
                       max_features_to_plot: int = 20,
                       fontsize: int = 14):
@@ -180,7 +189,7 @@ def compute_vif_multi(input_directory: str,
         input_directory (str): Target directory with CSV files able to be loaded as DataFrame.
         output_plot_directory (str): Save plots to this directory.
         output_dataset_directory (str | None): If provided, saves new CSV files to this directory.
-        target_columns (list[str] | None): Optional list of columns to include. Defaults to all numeric columns.
+        use_columns (list[str] | None): Optional list of columns to include. Defaults to all numeric columns.
         ignore_columns (list[str] | None): Optional list of columns to exclude from the VIF computation. Skipped if `target_columns` is provided.
         max_features_to_plot (int): Adjust the number of features shown in the plot.
         fontsize (int): Base fontsize to scale title and labels on hte plot.
@@ -195,7 +204,7 @@ def compute_vif_multi(input_directory: str,
     
     for df, df_name in yield_dataframes_from_dir(datasets_dir=input_directory):
         vif_dataframe = compute_vif(df=df,
-                            target_columns=target_columns,
+                            use_columns=use_columns,
                             ignore_columns=ignore_columns,
                             max_features_to_plot=max_features_to_plot,
                             fontsize=fontsize,
@@ -205,5 +214,11 @@ def compute_vif_multi(input_directory: str,
         
         if output_dataset_directory is not None:
             new_filename = 'VIF_' + df_name
-            result_df = drop_vif_based(df=df, vif_df=vif_dataframe)
-            save_dataframe(df=result_df, save_dir=output_dataset_directory, filename=new_filename)
+            result_df, dropped_cols = drop_vif_based(df=df, vif_df=vif_dataframe)
+            
+            if len(dropped_cols) > 0:
+                save_dataframe(df=result_df, save_dir=output_dataset_directory, filename=new_filename)
+
+
+def info():
+    _script_info(__all__)

ml_tools/data_exploration.py

@@ -9,22 +9,23 @@ from typing import Union, Literal, Dict, Tuple
 import os
 import sys
 import textwrap
-from ml_tools.utilities import sanitize_filename
+from ml_tools.utilities import sanitize_filename, _script_info
 
 
-# Keep track of all available functions, show using `info()`
-__all__ = ["summarize_dataframe",
-           "drop_rows_with_missing_data",
-           "split_features_targets", 
-           "show_null_columns",
-           "drop_columns_with_missing_data",
-           "split_continuous_binary",
-           "plot_correlation_heatmap",
-           "check_value_distributions",
-           "plot_value_distributions",
-           "clip_outliers_single",
-           "clip_outliers_multi",
-           "merge_dataframes"]
+# Keep track of all available tools, show using `info()`
+__all__ = [
+    "summarize_dataframe",
+    "drop_rows_with_missing_data",
+    "split_features_targets", 
+    "show_null_columns",
+    "drop_columns_with_missing_data",
+    "split_continuous_binary", 
+    "plot_correlation_heatmap", 
+    "check_value_distributions", 
+    "plot_value_distributions", 
+    "clip_outliers_single", 
+    "clip_outliers_multi"
+]
 
 
 def summarize_dataframe(df: pd.DataFrame, round_digits: int = 2):
@@ -58,34 +59,6 @@ def summarize_dataframe(df: pd.DataFrame, round_digits: int = 2):
     return summary
 
 
-def show_null_columns(df: pd.DataFrame, round_digits: int = 2):
-    """
-    Displays a table of columns with missing values, showing both the count and
-    percentage of missing entries per column.
-
-    Parameters:
-        df (pd.DataFrame): The input DataFrame.
-        round_digits (int): Number of decimal places for the percentage.
-        
-    Returns:
-        pd.DataFrame: A DataFrame summarizing missing values in each column.
-    """
-    null_counts = df.isnull().sum()
-    null_percent = df.isnull().mean() * 100
-
-    # Filter only columns with at least one null
-    mask = null_counts > 0
-    null_summary = pd.DataFrame({
-        'Missing Count': null_counts[mask],
-        'Missing %': null_percent[mask].round(round_digits)
-    })
-
-    # Sort by descending percentage of missing values
-    null_summary = null_summary.sort_values(by='Missing %', ascending=False)
-    # print(null_summary)
-    return null_summary
-
-
 def drop_rows_with_missing_data(df: pd.DataFrame, threshold: float = 0.7) -> pd.DataFrame:
     """
     Drops rows with more than `threshold` fraction of missing values.
@@ -132,6 +105,57 @@ def split_features_targets(df: pd.DataFrame, targets: list[str]):
     return df_targets, df_features
 
 
+def show_null_columns(df: pd.DataFrame, round_digits: int = 2):
+    """
+    Displays a table of columns with missing values, showing both the count and
+    percentage of missing entries per column.
+
+    Parameters:
+        df (pd.DataFrame): The input DataFrame.
+        round_digits (int): Number of decimal places for the percentage.
+        
+    Returns:
+        pd.DataFrame: A DataFrame summarizing missing values in each column.
+    """
+    null_counts = df.isnull().sum()
+    null_percent = df.isnull().mean() * 100
+
+    # Filter only columns with at least one null
+    mask = null_counts > 0
+    null_summary = pd.DataFrame({
+        'Missing Count': null_counts[mask],
+        'Missing %': null_percent[mask].round(round_digits)
+    })
+
+    # Sort by descending percentage of missing values
+    null_summary = null_summary.sort_values(by='Missing %', ascending=False)
+    # print(null_summary)
+    return null_summary
+
+
+def drop_columns_with_missing_data(df: pd.DataFrame, threshold: float = 0.7) -> pd.DataFrame:
+    """
+    Drops columns with more than `threshold` fraction of missing values.
+
+    Parameters:
+        df (pd.DataFrame): The input DataFrame.
+        threshold (float): Fraction of missing values above which columns are dropped.
+
+    Returns:
+        pd.DataFrame: A new DataFrame without the dropped columns.
+    """
+    missing_fraction = df.isnull().mean()
+    cols_to_drop = missing_fraction[missing_fraction > threshold].index
+
+    if len(cols_to_drop) > 0:
+        print(f"Dropping columns with more than {threshold*100:.0f}% missing data:")
+        print(list(cols_to_drop))
+    else:
+        print(f"No columns have more than {threshold*100:.0f}% missing data.")
+
+    return df.drop(columns=cols_to_drop)
+
+
 def split_continuous_binary(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
     """
     Split DataFrame into two DataFrames: one with continuous columns, one with binary columns.
@@ -174,29 +198,6 @@ def split_continuous_binary(df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFram
 
     return df_cont, df_bin # type: ignore
 
-    
-def drop_columns_with_missing_data(df: pd.DataFrame, threshold: float = 0.7) -> pd.DataFrame:
-    """
-    Drops columns with more than `threshold` fraction of missing values.
-
-    Parameters:
-        df (pd.DataFrame): The input DataFrame.
-        threshold (float): Fraction of missing values above which columns are dropped.
-
-    Returns:
-        pd.DataFrame: A new DataFrame without the dropped columns.
-    """
-    missing_fraction = df.isnull().mean()
-    cols_to_drop = missing_fraction[missing_fraction > threshold].index
-
-    if len(cols_to_drop) > 0:
-        print(f"Dropping columns with more than {threshold*100:.0f}% missing data:")
-        print(list(cols_to_drop))
-    else:
-        print(f"No columns have more than {threshold*100:.0f}% missing data.")
-
-    return df.drop(columns=cols_to_drop)
-
 
 def plot_correlation_heatmap(df: pd.DataFrame, save_dir: Union[str, None] = None, method: Literal["pearson", "kendall", "spearman"]="pearson", plot_title: str="Correlation Heatmap"):
     """
@@ -513,83 +514,10 @@ def clip_outliers_multi(
     return new_df
 
 
-def merge_dataframes(
-    *dfs: pd.DataFrame,
-    reset_index: bool = False,
-    direction: Literal["horizontal", "vertical"] = "horizontal"
-) -> pd.DataFrame:
-    """
-    Merges multiple DataFrames either horizontally or vertically.
-
-    Parameters:
-        *dfs (pd.DataFrame): Variable number of DataFrames to merge.
-        reset_index (bool): Whether to reset index in the final merged DataFrame.
-        direction (["horizontal" | "vertical"]):
-            - "horizontal": Merge on index, adding columns.
-            - "vertical": Append rows; all DataFrames must have identical columns.
-
-    Returns:
-        pd.DataFrame: A single merged DataFrame.
-
-    Raises:
-        ValueError:
-            - If fewer than 2 DataFrames are provided.
-            - If indexes do not match for horizontal merge.
-            - If column names or order differ for vertical merge.
-    """
-    if len(dfs) < 2:
-        raise ValueError("At least 2 DataFrames must be provided.")
-    
-    for i, df in enumerate(dfs, start=1):
-        print(f"DataFrame {i} shape: {df.shape}")
-    
-
-    if direction == "horizontal":
-        reference_index = dfs[0].index
-        for i, df in enumerate(dfs, start=1):
-            if not df.index.equals(reference_index):
-                raise ValueError(f"Indexes do not match: Dataset 1 and Dataset {i}.")
-        merged_df = pd.concat(dfs, axis=1)
-
-    elif direction == "vertical":
-        reference_columns = dfs[0].columns
-        for i, df in enumerate(dfs, start=1):
-            if not df.columns.equals(reference_columns):
-                raise ValueError(f"Column names/order do not match: Dataset 1 and Dataset {i}.")
-        merged_df = pd.concat(dfs, axis=0)
-
-    else:
-        raise ValueError(f"Invalid merge direction: {direction}")
-
-    if reset_index:
-        merged_df = merged_df.reset_index(drop=True)
-
-    print(f"Merged DataFrame shape: {merged_df.shape}")
-
-    return merged_df
-
-
 def _is_notebook():
     return get_ipython() is not None
 
 
-def info(full_info: bool=True):
-    """
-    List available functions and their descriptions.
-    """
-    print("Available functions for data exploration:")
-    if full_info:
-        module = sys.modules[__name__]
-        for name in __all__:
-            obj = getattr(module, name, None)
-            if callable(obj):
-                doc = obj.__doc__ or "No docstring provided."
-                formatted_doc = textwrap.indent(textwrap.dedent(doc.strip()), prefix="    ")
-                print(f"\n{name}:\n{formatted_doc}")
-    else:
-        for i, name in enumerate(__all__, start=1):
-            print(f"{i} - {name}")
-
+def info():
+    _script_info(__all__)
 
-if __name__ == "__main__":
-    info()

ml_tools/datasetmaster.py

@@ -11,6 +11,15 @@ from PIL import Image
 from torchvision.datasets import ImageFolder
 from torchvision import transforms
 import matplotlib.pyplot as plt
+from .utilities import _script_info
+
+
+__all__ = [
+    "DatasetMaker",
+    "PytorchDataset",
+    "make_vision_dataset",
+    "SequenceDataset",
+]
 
 
 class DatasetMaker():
@@ -592,4 +601,7 @@ class SequenceDataset():
         
     def __len__(self):
         return f"Train: {len(self.train_dataset)}, Test: {len(self.test_dataset)}"
-        
+
+
+def info():
+    _script_info(__all__)