dragon-ml-toolbox 1.1.2__py3-none-any.whl

dragon_ml_toolbox-1.1.2.dist-info/METADATA

@@ -0,0 +1,114 @@
+Metadata-Version: 2.4
+Name: dragon-ml-toolbox
+Version: 1.1.2
+Summary: A collection of tools for machine learning projects
+Author-email: Karl Loza <luigiloza@gmail.com>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Provides-Extra: data-exploration
+Requires-Dist: pandas; extra == "data-exploration"
+Requires-Dist: numpy; extra == "data-exploration"
+Requires-Dist: matplotlib; extra == "data-exploration"
+Requires-Dist: seaborn; extra == "data-exploration"
+Requires-Dist: statsmodels; extra == "data-exploration"
+Requires-Dist: ipython; extra == "data-exploration"
+Provides-Extra: datasetmaster
+Requires-Dist: torch; extra == "datasetmaster"
+Requires-Dist: pandas; extra == "datasetmaster"
+Requires-Dist: numpy; extra == "datasetmaster"
+Requires-Dist: scikit-learn; extra == "datasetmaster"
+Requires-Dist: imblearn; extra == "datasetmaster"
+Requires-Dist: Pillow; extra == "datasetmaster"
+Requires-Dist: matplotlib; extra == "datasetmaster"
+Provides-Extra: ensemble-learning
+Requires-Dist: pandas; extra == "ensemble-learning"
+Requires-Dist: numpy; extra == "ensemble-learning"
+Requires-Dist: seaborn; extra == "ensemble-learning"
+Requires-Dist: matplotlib; extra == "ensemble-learning"
+Requires-Dist: joblib; extra == "ensemble-learning"
+Requires-Dist: imblearn; extra == "ensemble-learning"
+Requires-Dist: scikit-learn; extra == "ensemble-learning"
+Requires-Dist: xgboost; extra == "ensemble-learning"
+Requires-Dist: lightgbm; extra == "ensemble-learning"
+Requires-Dist: shap; extra == "ensemble-learning"
+Provides-Extra: handle-excel
+Requires-Dist: openpyxl; extra == "handle-excel"
+Requires-Dist: pandas; extra == "handle-excel"
+Provides-Extra: logger
+Requires-Dist: pandas; extra == "logger"
+Requires-Dist: openpyxl; extra == "logger"
+Provides-Extra: mice-imputation
+Requires-Dist: pandas; extra == "mice-imputation"
+Requires-Dist: miceforest; extra == "mice-imputation"
+Requires-Dist: matplotlib; extra == "mice-imputation"
+Requires-Dist: numpy; extra == "mice-imputation"
+Provides-Extra: particle-swarm-optimization
+Requires-Dist: numpy; extra == "particle-swarm-optimization"
+Requires-Dist: joblib; extra == "particle-swarm-optimization"
+Requires-Dist: xgboost; extra == "particle-swarm-optimization"
+Requires-Dist: lightgbm; extra == "particle-swarm-optimization"
+Requires-Dist: scikit-learn; extra == "particle-swarm-optimization"
+Requires-Dist: polars; extra == "particle-swarm-optimization"
+Provides-Extra: pytorch-models
+Requires-Dist: torch; extra == "pytorch-models"
+Provides-Extra: trainer
+Requires-Dist: numpy; extra == "trainer"
+Requires-Dist: torch; extra == "trainer"
+Requires-Dist: matplotlib; extra == "trainer"
+Requires-Dist: scikit-learn; extra == "trainer"
+Provides-Extra: utilities
+Requires-Dist: numpy; extra == "utilities"
+Requires-Dist: pandas; extra == "utilities"
+Provides-Extra: vision-helpers
+Requires-Dist: Pillow; extra == "vision-helpers"
+Requires-Dist: torch; extra == "vision-helpers"
+Requires-Dist: torchvision; extra == "vision-helpers"
+
+# ml_tools
+
+A collection of Python utilities and machine learning tools, structured as a modular package for easy reuse and installation.
+
+## Features
+
+- Modular scripts for data exploration, logging, machine learning, and more.
+- Optional dependencies grouped by functionality for lightweight installs.
+- Designed for seamless integration as a Git submodule or installable Python package.
+
+
+## Installation
+
+### Via GitHub (Editable / Development Mode)
+
+Clone the repository and install in editable mode with optional dependencies:
+
+```bash
+git clone https://github.com/DrAg0n-BoRn/ML_tools.git
+cd ML_tools
+pip install -e '.[utilities]'
+```
+
+### Via PyPI (Stable Releases)
+
+Install the latest stable release from PyPI with optional dependencies:
+
+pip install ml_tools[utilities]
+
+## Usage
+
+After installation, import modules like this:
+
+```python
+from ml_tools.utilities import sanitize_filename
+from ml_tools.logger import custom_logger
+```
+
+## Development
+
+Python 3.9+ recommended.
+
+To install all dependencies including development tools:
+
+```python
+pip install -e '.[dev]'
+```
+

dragon_ml_toolbox-1.1.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+ml_tools/MICE_imputation.py,sha256=wLM4DJTs-CxuGzEmuTj7Tmb7AoKGs16cdxQD2Ne8Dv0,7340
+ml_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ml_tools/data_exploration.py,sha256=Nx8V6xYmh2XqMF3WXg0BdAQnDAFq5cFd36JHFIf56vc,26989
+ml_tools/datasetmaster.py,sha256=VUneKshnmjOGbtqVVGTFcIMRKF3s6ZDYrosIYKDjD80,28956
+ml_tools/ensemble_learning.py,sha256=uA7A94CLv8o2l125oTEi0cjHusZkB-7Mnrtn7SGTfjs,29714
+ml_tools/handle_excel.py,sha256=IR0VQc3hYdmjwC31E5YxDnRcWig4jSIx7Y_7to-KZz4,11969
+ml_tools/logger.py,sha256=yQ5v8e2UnkKgQDszpg5zihpLPI8ehEci7p_2PKkshls,4613
+ml_tools/particle_swarm_optimization.py,sha256=jpkje4OETC9fyISxxUTx4XGrImSU6gDEcwz46ZDs2bQ,19250
+ml_tools/pytorch_models.py,sha256=Oykw02sOZLCjvSadQd64UGesBN7kq0x1EGXHusvYiQI,9908
+ml_tools/trainer.py,sha256=Zd7AaHeoNd8dEas2JChWoHaCUpWUVRDUMybuHaKJ0XY,16740
+ml_tools/utilities.py,sha256=mG_--EFplfI9H7OhrWI8VkdNJtTbs4Wbz32xvcFWps8,5518
+ml_tools/vision_helpers.py,sha256=lBAW6dzAK-HOswAt1fU_tfP9hkNLY5D8c_I_7hhEXno,7528
+dragon_ml_toolbox-1.1.2.dist-info/METADATA,sha256=PaAoR_VbZS6USH1kzYFkWM6DqXEP_mDsCtprwFN7IIs,4076
+dragon_ml_toolbox-1.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dragon_ml_toolbox-1.1.2.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
+dragon_ml_toolbox-1.1.2.dist-info/RECORD,,

dragon_ml_toolbox-1.1.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dragon_ml_toolbox-1.1.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+ml_tools

ml_tools/MICE_imputation.py

@@ -0,0 +1,178 @@
+import pandas as pd
+import miceforest as mf
+import os
+import matplotlib.pyplot as plt
+import numpy as np
+from utilities import load_dataframe, list_csv_paths
+
+
+def apply_mice(df: pd.DataFrame, df_name: str, resulting_datasets: int=1, iterations: int=20, random_state: int=101):
+    
+    # Initialize kernel with number of imputed datasets to generate
+    kernel = mf.ImputationKernel(
+        data=df,
+        datasets=resulting_datasets,
+        random_state=random_state
+    )
+    
+    # Perform MICE with n iterations per dataset
+    kernel.mice(iterations)
+    
+    # Retrieve the imputed datasets 
+    imputed_datasets = [kernel.complete_data(dataset=i) for i in range(resulting_datasets)]
+    
+    if resulting_datasets == 1:
+        imputed_dataset_names = [f"{df_name}_imputed"]
+    else:
+        imputed_dataset_names = [f"{df_name}_imputed_{i+1}" for i in range(resulting_datasets)]
+    
+    # Ensure indexes match
+    for imputed_df, subname in zip(imputed_datasets, imputed_dataset_names):
+        assert imputed_df.shape[0] == df.shape[0], f"Row count mismatch in dataset {subname}"
+        assert all(imputed_df.index == df.index), f"Index mismatch in dataset {subname}"
+    # print("✅ All imputed datasets match the original DataFrame indexes.")
+    
+    return kernel, imputed_datasets, imputed_dataset_names
+
+
+def save_imputed_datasets(save_dir: str, imputed_datasets: list, imputed_dataset_names: list[str]):
+    # Check path
+    os.makedirs(save_dir, exist_ok=True)
+    
+    for imputed_df, subname in zip(imputed_datasets, imputed_dataset_names):
+        output_path = os.path.join(save_dir, subname + ".csv")
+        imputed_df.to_csv(output_path, index=False, encoding='utf-8')
+        print(f"\tSaved {subname} with shape {imputed_df.shape}")
+        
+
+#Get names of features that had missing values before imputation
+def get_na_column_names(df: pd.DataFrame):
+    return [col for col in df.columns if df[col].isna().any()]
+
+
+#Convergence diagnostic
+def get_convergence_diagnostic(kernel: mf.ImputationKernel, imputed_dataset_names: list[str], column_names: list[str], root_dir: str):
+    # get number of iterations used
+    iterations_cap = kernel.iteration_count()
+    
+    # Check path
+    os.makedirs(root_dir, exist_ok=True)
+    
+    # iterate over each imputed dataset
+    for dataset_id, imputed_dataset_name in zip(range(kernel.num_datasets), imputed_dataset_names):
+        #Check directory for current dataset
+        dataset_file_dir = f"Convergence_Metrics_{imputed_dataset_name}"
+        local_save_dir = os.path.join(root_dir, dataset_file_dir)
+        if not os.path.isdir(local_save_dir):
+            os.makedirs(local_save_dir)
+        
+        for feature_name in column_names:
+            means_per_iteration = []
+            for iteration in range(iterations_cap):
+                current_imputed = kernel.complete_data(dataset=dataset_id, iteration=iteration)
+                means_per_iteration.append(np.mean(current_imputed[feature_name]))
+
+            plt.plot(means_per_iteration, marker='o')
+            plt.xlabel("Iteration")
+            plt.ylabel("Mean of Imputed Values")
+            plt.title(f"Mean Convergence for '{feature_name}'")
+            
+            # Adjust plot display for the X axis
+            _ticks = np.arange(iterations_cap)
+            _labels = np.arange(1, iterations_cap + 1)
+            plt.xticks(ticks=_ticks, labels=_labels)
+            
+            save_path = os.path.join(local_save_dir, feature_name + ".svg")
+            plt.savefig(save_path, bbox_inches='tight', format="svg")
+            plt.close()
+            
+        print(f"{dataset_file_dir} completed.")
+
+
+# Imputed distributions
+def get_imputed_distributions(kernel: mf.ImputationKernel, df_name: str, root_dir: str, column_names: list[str], one_plot: bool=False, fontsize: int=18):
+    ''' 
+    It works using miceforest's authors implementation of the method `.plot_imputed_distributions()`.
+    
+    Set `one_plot=True` to save a single image including all feature distribution plots instead.
+    '''
+    # Check path
+    os.makedirs(root_dir, exist_ok=True)
+    local_save_dir = os.path.join(root_dir, f"Distribution_Metrics_{df_name}")
+    if not os.path.isdir(local_save_dir):
+        os.makedirs(local_save_dir)
+    
+    # Styling parameters
+    legend_kwargs = {'frameon': True, 'facecolor': 'white', 'framealpha': 0.8}
+    label_font = {'size': fontsize, 'weight': 'bold'}
+
+    def _process_figure(fig, filename):
+        """Helper function to add labels and legends to a figure"""
+        for ax in fig.axes:
+            # Set axis labels
+            ax.set_xlabel('Value', **label_font)
+            ax.set_ylabel('Density', **label_font)
+            
+            # Add legend based on line colors
+            lines = ax.get_lines()
+            if len(lines) >= 1:
+                lines[0].set_label('Original Data')
+                if len(lines) > 1:
+                    lines[1].set_label('Imputed Data')
+                ax.legend(**legend_kwargs)
+                
+        # Adjust layout and save
+        fig.tight_layout()
+        fig.savefig(
+            os.path.join(local_save_dir, filename),
+            format='svg',
+            bbox_inches='tight',
+            pad_inches=0
+        )
+        plt.close(fig)
+
+    if one_plot:
+        # Generate combined plot
+        fig = kernel.plot_imputed_distributions(variables=column_names)
+        _process_figure(fig, "Combined_Distributions.svg")
+        # Generate individual plots per feature
+    else:
+        for feature in column_names:
+            fig = kernel.plot_imputed_distributions(variables=[feature])
+            _process_figure(fig, f"{feature}.svg")
+
+    print("Imputed distributions saved successfully.")
+
+
+def run_mice_pipeline(df_path_or_dir: str, save_datasets_dir: str, save_metrics_dir: str, resulting_datasets: int=1, iterations: int=20, random_state: int=101):
+    """
+    Call functions in sequence for each dataset in the provided path or directory:
+        1. Load dataframe
+        2. Apply MICE
+        3. Save imputed dataset(s)
+        4. Save convergence metrics
+        5. Save distribution metrics
+    """
+    # Check paths
+    os.makedirs(save_datasets_dir, exist_ok=True)
+    os.makedirs(save_metrics_dir, exist_ok=True)
+    
+    if os.path.isfile(df_path_or_dir):
+        all_file_paths = [df_path_or_dir]
+    elif os.path.isdir(df_path_or_dir):
+        all_file_paths, _ = list_csv_paths(df_path_or_dir)
+    else:
+        raise ValueError(f"Invalid path or directory: {df_path_or_dir}")
+    
+    for df_path in all_file_paths:
+        df, df_name = load_dataframe(df_path=df_path)
+        
+        kernel, imputed_datasets, imputed_dataset_names = apply_mice(df=df, df_name=df_name, resulting_datasets=resulting_datasets, iterations=iterations, random_state=random_state)
+        
+        save_imputed_datasets(save_dir=save_datasets_dir, imputed_datasets=imputed_datasets, imputed_dataset_names=imputed_dataset_names)
+        
+        imputed_column_names = get_na_column_names(df=df)
+        
+        get_convergence_diagnostic(kernel=kernel, imputed_dataset_names=imputed_dataset_names, column_names=imputed_column_names, root_dir=save_metrics_dir)
+        
+        get_imputed_distributions(kernel=kernel, df_name=df_name, root_dir=save_metrics_dir, column_names=imputed_column_names)