dragon-ml-toolbox 8.1.0__py3-none-any.whl → 9.0.0__py3-none-any.whl

ml_tools/utilities.py

@@ -76,11 +76,13 @@ def load_dataframe(
             df = pl.read_csv(path, infer_schema_length=1000)
             
     else:
-        raise ValueError(f"❌ Invalid kind '{kind}'. Must be one of 'pandas' or 'polars'.")
+        _LOGGER.error(f"Invalid kind '{kind}'. Must be one of 'pandas' or 'polars'.")
+        raise ValueError()
 
     # This check works for both pandas and polars DataFrames
     if df.shape[0] == 0:
-        raise ValueError(f"❌ DataFrame '{df_name}' loaded from '{path}' is empty.")
+        _LOGGER.error(f"DataFrame '{df_name}' loaded from '{path}' is empty.")
+        raise ValueError()
 
     if verbose:
         _LOGGER.info(f"💾 Loaded {kind.upper()} dataset: '{df_name}' with shape: {df.shape}")
@@ -162,13 +164,14 @@ def merge_dataframes(
         merged_df = pd.concat(dfs, axis=0)
 
     else:
-        raise ValueError(f"❌ Invalid merge direction: {direction}")
+        _LOGGER.error(f"Invalid merge direction: {direction}")
+        raise ValueError()
 
     if reset_index:
         merged_df = merged_df.reset_index(drop=True)
     
     if verbose:
-        _LOGGER.info(f"✅ Merged DataFrame shape: {merged_df.shape}")
+        _LOGGER.info(f"Merged DataFrame shape: {merged_df.shape}")
 
     return merged_df
 
@@ -187,7 +190,7 @@ def save_dataframe(df: Union[pd.DataFrame, pl.DataFrame], save_dir: Union[str,Pa
     """
     # This check works for both pandas and polars
     if df.shape[0] == 0:
-        _LOGGER.warning(f"⚠️ Attempting to save an empty DataFrame: '{filename}'. Process Skipped.")
+        _LOGGER.warning(f"Attempting to save an empty DataFrame: '{filename}'. Process Skipped.")
         return
     
     # Create the directory if it doesn't exist
@@ -207,9 +210,10 @@ def save_dataframe(df: Union[pd.DataFrame, pl.DataFrame], save_dir: Union[str,Pa
         df.write_csv(output_path) # Polars defaults to utf8 and no index
     else:
         # This error handles cases where an unsupported type is passed
-        raise TypeError(f"❌ Unsupported DataFrame type: {type(df)}. Must be pandas or polars.")
+        _LOGGER.error(f"Unsupported DataFrame type: {type(df)}. Must be pandas or polars.")
+        raise TypeError()
     
-    _LOGGER.info(f"✅ Saved dataset: '{filename}' with shape: {df.shape}")
+    _LOGGER.info(f"Saved dataset: '{filename}' with shape: {df.shape}")
 
 
 def normalize_mixed_list(data: list, threshold: int = 2) -> list[float]:
@@ -243,7 +247,8 @@ def normalize_mixed_list(data: list, threshold: int = 2) -> list[float]:
     
     # Raise for negative values
     if any(x < 0 for x in float_list):
-        raise ValueError("❌ Negative values are not allowed in the input list.")
+        _LOGGER.error("Negative values are not allowed in the input list.")
+        raise ValueError()
     
     # Step 2: Compute log10 of non-zero values
     nonzero = [x for x in float_list if x > 0]
@@ -302,14 +307,16 @@ def threshold_binary_values(
     elif isinstance(input_array, (list, tuple)):
         array = np.array(input_array)
     else:
-        raise TypeError("❌ Unsupported input type")
+        _LOGGER.error("Unsupported input type")
+        raise TypeError()
 
     array = array.flatten()
     total = array.shape[0]
 
     bin_count = total if binary_values is None else binary_values
     if not (0 <= bin_count <= total):
-        raise ValueError("❌ binary_values must be between 0 and the total number of elements")
+        _LOGGER.error("'binary_values' must be between 0 and the total number of elements")
+        raise ValueError()
 
     if bin_count == 0:
         result = array
@@ -349,9 +356,15 @@ def threshold_binary_values_batch(
     np.ndarray
         Thresholded array, same shape as input.
     """
-    assert input_array.ndim == 2, f"❌ Expected 2D array, got {input_array.ndim}D"
+    if input_array.ndim != 2:
+        _LOGGER.error(f"Expected 2D array, got {input_array.ndim}D array.")
+        raise AssertionError()
+    
     batch_size, total_features = input_array.shape
-    assert 0 <= binary_values <= total_features, "❌ binary_values out of valid range"
+    
+    if not (0 <= binary_values <= total_features):
+        _LOGGER.error("'binary_values' out of valid range.")
+        raise AssertionError()
 
     if binary_values == 0:
         return input_array.copy()
@@ -380,15 +393,13 @@ def serialize_object(obj: Any, save_dir: Union[str,Path], filename: str, verbose
         full_path = save_path / sanitized_name
         joblib.dump(obj, full_path)
     except (IOError, OSError, TypeError, TerminatedWorkerError) as e:
-        message = f"❌ Failed to serialize object of type '{type(obj)}': {e}"
+        _LOGGER.error(f"Failed to serialize object of type '{type(obj)}'.")
         if raise_on_error:
-            raise Exception(message)
-        else:
-            _LOGGER.warning(message)
+            raise e
         return None
     else:
         if verbose:
-            _LOGGER.info(f"✅ Object of type '{type(obj)}' saved to '{full_path}'")
+            _LOGGER.info(f"Object of type '{type(obj)}' saved to '{full_path}'")
         return None
 
 
@@ -407,15 +418,13 @@ def deserialize_object(filepath: Union[str,Path], verbose: bool=True, raise_on_e
     try:
         obj = joblib.load(true_filepath)
     except (IOError, OSError, EOFError, TypeError, ValueError) as e:
-        message = f"❌ Failed to deserialize object from '{true_filepath}': {e}"
+        _LOGGER.error(f"Failed to deserialize object from '{true_filepath}'.")
         if raise_on_error:
-            raise Exception(message)
-        else:
-            _LOGGER.warning(message)
+            raise e
         return None
     else:
         if verbose:
-            _LOGGER.info(f"✅ Loaded object of type '{type(obj)}'")
+            _LOGGER.info(f"Loaded object of type '{type(obj)}'.")
         return obj
 
 
@@ -486,7 +495,8 @@ def train_dataset_orchestrator(list_of_dirs: list[Union[str,Path]],
     for dir in list_of_dirs:
         dir_path = make_fullpath(dir)
         if not dir_path.is_dir():
-            raise IOError(f"'{dir}' is not a directory.")
+            _LOGGER.error(f"'{dir}' is not a directory.")
+            raise IOError()
         all_dir_paths.append(dir_path)
     
     # main loop
@@ -502,10 +512,10 @@ def train_dataset_orchestrator(list_of_dirs: list[Union[str,Path]],
                     save_dataframe(df=df, save_dir=save_dir, filename=filename)
                     total_saved += 1
             except Exception as e:
-                _LOGGER.warning(f"⚠️ Failed to process file '{df_path}'. Reason: {e}")
+                _LOGGER.error(f"Failed to process file '{df_path}'. Reason: {e}")
                 continue 
 
-    _LOGGER.info(f"✅ {total_saved} single-target datasets were created.")
+    _LOGGER.info(f"{total_saved} single-target datasets were created.")
 
 
 def train_dataset_yielder(
@@ -530,6 +540,5 @@ def train_dataset_yielder(
         yield (df_features, df_target, feature_names, target_col)
 
 
-
 def info():
     _script_info(__all__)

dragon_ml_toolbox-8.1.0.dist-info/RECORD

@@ -1,36 +0,0 @@
-dragon_ml_toolbox-8.1.0.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
-dragon_ml_toolbox-8.1.0.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=lY4_rJPnLnMu7YBQaY-_iz1JRDcLdQzNCyeLAF1glJY,1837
-ml_tools/ETL_engineering.py,sha256=4wwZXi9_U7xfCY70jGBaKniOeZ0m75ppxWpQBd_DmLc,39369
-ml_tools/GUI_tools.py,sha256=n4ZZ5kEjwK5rkOCFJE41HeLFfjhpJVLUSzk9Kd9Kr_0,45410
-ml_tools/MICE_imputation.py,sha256=oFHg-OytOzPYTzBR_wIRHhP71cMn3aupDeT59ABsXlQ,11576
-ml_tools/ML_callbacks.py,sha256=noedVMmHZ72Odbg28zqx5wkhhvX2v-jXicKE_NCAiqU,13838
-ml_tools/ML_datasetmaster.py,sha256=tN-GBPEwXRWFBT8r8K0v9b3Bd77DhqSH5FkjDP6BHTw,28847
-ml_tools/ML_evaluation.py,sha256=BER5dOvSTySNzO92gm8tIpqJ5vT-s0iHMmaoly1uUH8,16018
-ml_tools/ML_evaluation_multi.py,sha256=uVtKGYWgOLv34Xj_jz6E_HAYzNb0HwRbMwA8oFZWpUk,12395
-ml_tools/ML_inference.py,sha256=hwtAdyDCE1xtqLgJgyOTAPck0eTmkOCJK1cM_IJSdck,22824
-ml_tools/ML_models.py,sha256=xZiSFh7S6eitl-VjjvNpsikojDvurK8n_ueLEh6_5pM,27979
-ml_tools/ML_optimization.py,sha256=GX-qZ2mCI3gWRCTP5w7lXrZpfGle3J_mE0O68seIoio,13475
-ml_tools/ML_scaler.py,sha256=pGkp1nUpeuoBvbq5hUkieQdxex6kNef1mEbeS_HUCJs,7471
-ml_tools/ML_trainer.py,sha256=6JSmEQaCPSo-S_5plNBTPw-SYgzZpyMNwiqpShJf7qU,23726
-ml_tools/PSO_optimization.py,sha256=9Y074d-B5h4Wvp9YPiy6KAeXM-Yv6Il3gWalKvOLVgo,22705
-ml_tools/RNN_forecast.py,sha256=2CyjBLSYYc3xLHxwLXUmP5Qv8AmV1OB_EndETNX1IBk,1956
-ml_tools/SQL.py,sha256=bkSTmMV4CtEqa67hApYWaRxTqwAlKIc5_b28P1bnDwg,10475
-ml_tools/VIF_factor.py,sha256=2nUMupfUoogf8o6ghoFZk_OwWhFXU0R3C9Gj0HOlI14,10415
-ml_tools/_ML_optimization_multi.py,sha256=DrNG3Vf1uUw-3CpYfXREgSGuR4dTpLWY1F3R9j-PYqQ,9816
-ml_tools/__init__.py,sha256=q0y9faQ6e17XCQ7eUiCZ1FJ4Bg5EQqLjZ9f_l5REUUY,41
-ml_tools/_logger.py,sha256=TpgYguxO-CWYqqgLW0tqFjtwZ58PE_W2OCfWNGZr0n0,1175
-ml_tools/_script_info.py,sha256=21r83LV3RubsNZ_RTEUON6RbDf7Mh4_udweNcvdF_Fk,212
-ml_tools/custom_logger.py,sha256=nyLRxaRxkqYOFdSjI0X2BWXB8C2IU18QfmqIFKqSedI,5820
-ml_tools/data_exploration.py,sha256=RuMHWagXrSQi1MzAMlYeBeVg7UxhVvEq8gJ9bIam2BM,27103
-ml_tools/ensemble_evaluation.py,sha256=wnqoTPg4WYWf2A8z5XT0eSlW4snEuLCXQVj88sZKzQ4,24683
-ml_tools/ensemble_inference.py,sha256=rtU7eUaQne615n2g7IHZCJI-OvrBCcjxbTkEIvtCGFQ,9414
-ml_tools/ensemble_learning.py,sha256=dAyFgSTyvxJWjc_enJ_8EUoWwiekBeoNyJNxVY-kcUU,21868
-ml_tools/handle_excel.py,sha256=J9iwIqMZemoxK49J5osSwp9Ge0h9YTKyYGbOm53hcno,13007
-ml_tools/keys.py,sha256=HtPG8-MWh89C32A7eIlfuuA-DLwkxGkoDfwR2TGN9CQ,1074
-ml_tools/optimization_tools.py,sha256=EL5tgNFwRo-82pbRE1CFVy9noNhULD7wprWuKadPheg,5090
-ml_tools/path_manager.py,sha256=Z8e7w3MPqQaN8xmTnKuXZS6CIW59BFwwqGhGc00sdp4,13692
-ml_tools/utilities.py,sha256=LqXXTovaHbA5AOKRk6Ru6DgAPAM0wPfYU70kUjYBryo,19231
-dragon_ml_toolbox-8.1.0.dist-info/METADATA,sha256=qGTl4__H1ZsbyJHtExcDt14i8ziWXpEy2WaRAELPmTI,6778
-dragon_ml_toolbox-8.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dragon_ml_toolbox-8.1.0.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
-dragon_ml_toolbox-8.1.0.dist-info/RECORD,,

ml_tools/_ML_optimization_multi.py

@@ -1,231 +0,0 @@
-import pandas as pd
-import torch
-import numpy as np
-import evotorch
-from evotorch.algorithms import NSGA2
-from evotorch.logging import PandasLogger
-from typing import Literal, Union, Tuple, List, Optional, Any, Callable
-from pathlib import Path
-from tqdm.auto import trange
-from functools import partial
-from contextlib import nullcontext
-import matplotlib.pyplot as plt
-import seaborn as sns
-
-from .path_manager import make_fullpath, sanitize_filename
-from ._logger import _LOGGER
-from ._script_info import _script_info
-from .ML_inference import PyTorchInferenceHandlerMulti # Using the multi-target handler
-from .keys import PyTorchInferenceKeys
-from .utilities import threshold_binary_values, save_dataframe
-from .SQL import DatabaseManager # Added for SQL saving
-
-__all__ = [
-    "create_multi_objective_problem",
-    "run_multi_objective_optimization",
-    "plot_pareto_front"
-]
-
-
-def create_multi_objective_problem(
-    inference_handler: PyTorchInferenceHandlerMulti,
-    bounds: Tuple[List[float], List[float]],
-    binary_features: int,
-    objective_senses: Tuple[Literal["min", "max"], ...],
-    algorithm: Literal["NSGA2"] = "NSGA2",
-    population_size: int = 200,
-    **searcher_kwargs
-) -> Tuple[evotorch.Problem, Callable[[], Any]]:
-    """
-    Creates and configures an EvoTorch Problem and a Searcher for multi-objective optimization.
-
-    This function sets up a problem where the goal is to optimize multiple conflicting
-    objectives simultaneously, using an algorithm like NSGA2 to find the Pareto front.
-
-    Args:
-        inference_handler (PyTorchInferenceHandlerMulti): An initialized handler for the multi-target model.
-        bounds (tuple[list[float], list[float]]): Lower and upper bounds for the solution features.
-        binary_features (int): Number of binary features at the end of the feature vector.
-        objective_senses (Tuple[Literal["min", "max"], ...]): A tuple specifying the optimization
-            goal for each target (e.g., ("max", "min", "max")). The length of this tuple
-            must match the number of outputs from the model.
-        algorithm (str): The multi-objective search algorithm to use. Currently supports "NSGA2".
-        population_size (int): The number of solutions in each generation.
-        **searcher_kwargs: Additional keyword arguments for the search algorithm's constructor.
-
-    Returns:
-        A tuple containing the configured multi-objective Problem and the Searcher factory.
-    """
-    lower_bounds, upper_bounds = list(bounds[0]), list(bounds[1])
-
-    if binary_features > 0:
-        lower_bounds.extend([0.45] * binary_features)
-        upper_bounds.extend([0.55] * binary_features)
-
-    solution_length = len(lower_bounds)
-    device = inference_handler.device
-
-    def fitness_func(solution_tensor: torch.Tensor) -> torch.Tensor:
-        """
-        The fitness function for a multi-objective problem.
-        It returns the entire output tensor from the model. EvoTorch handles the rest.
-        """
-        # The handler returns a tensor of shape [batch_size, num_targets]
-        predictions = inference_handler.predict_batch(solution_tensor)[PyTorchInferenceKeys.PREDICTIONS]
-        return predictions
-
-    if algorithm == "NSGA2":
-        problem = evotorch.Problem(
-            objective_sense=objective_senses,
-            objective_func=fitness_func,
-            solution_length=solution_length,
-            bounds=(lower_bounds, upper_bounds),
-            device=device,
-            vectorized=True,
-            num_actors='max' # Use available CPU cores
-        )
-        SearcherClass = NSGA2
-        if 'popsize' not in searcher_kwargs:
-            searcher_kwargs['popsize'] = population_size
-    else:
-        raise ValueError(f"Unknown multi-objective algorithm '{algorithm}'.")
-
-    searcher_factory = partial(SearcherClass, problem, **searcher_kwargs)
-    return problem, searcher_factory
-
-
-def run_multi_objective_optimization(
-    problem: evotorch.Problem,
-    searcher_factory: Callable[[], Any],
-    num_generations: int,
-    run_name: str,
-    binary_features: int,
-    save_dir: Union[str, Path],
-    feature_names: List[str],
-    target_names: List[str],
-    save_format: Literal['csv', 'sqlite', 'both'] = 'csv',
-    verbose: bool = True
-):
-    """
-    Runs the multi-objective evolutionary optimization process to find the Pareto front.
-
-    This function executes a multi-objective algorithm (like NSGA2) and saves the
-    entire set of non-dominated solutions (the Pareto front) to the specified format(s).
-    It also generates and saves a plot of the Pareto front.
-
-    Args:
-        problem (evotorch.Problem): The configured multi-objective problem.
-        searcher_factory (Callable): A factory function to generate a fresh searcher instance.
-        num_generations (int): The number of generations to run the algorithm.
-        run_name (str): A name for this optimization run, used for filenames/table names.
-        binary_features (int): Number of binary features in the solution vector.
-        save_dir (str | Path): The directory where the result files will be saved.
-        feature_names (List[str]): Names of the solution features for labeling columns.
-        target_names (List[str]): Names of the target objectives for labeling columns.
-        save_format (str): The format to save results in ('csv', 'sqlite', or 'both').
-        verbose (bool): If True, attaches a logger and saves the evolution history.
-    """
-    save_path = make_fullpath(save_dir, make=True, enforce="directory")
-    sanitized_run_name = sanitize_filename(run_name)
-    
-    if len(target_names) != problem.num_objectives:
-        raise ValueError("The number of `target_names` must match the number of objectives in the problem.")
-
-    searcher = searcher_factory()
-    _LOGGER.info(f"🤖 Starting multi-objective optimization with {searcher.__class__.__name__} for {num_generations} generations...")
-
-    logger = PandasLogger(searcher) if verbose else None
-    searcher.run(num_generations)
-
-    pareto_front = searcher.status["pareto_front"]
-    _LOGGER.info(f"✅ Optimization complete. Found {len(pareto_front)} non-dominated solutions.")
-
-    solutions_np = pareto_front.values.cpu().numpy()
-    objectives_np = pareto_front.evals.cpu().numpy()
-
-    if binary_features > 0:
-        solutions_np = threshold_binary_values(input_array=solutions_np, binary_values=binary_features)
-
-    results_df = pd.DataFrame(solutions_np, columns=feature_names)
-    objective_cols = []
-    for i, name in enumerate(target_names):
-        col_name = f"predicted_{name}"
-        results_df[col_name] = objectives_np[:, i]
-        objective_cols.append(col_name)
-
-    # --- Saving Logic ---
-    if save_format in ['csv', 'both']:
-        csv_path = save_path / f"pareto_front_{sanitized_run_name}.csv"
-        results_df.to_csv(csv_path, index=False)
-        _LOGGER.info(f"📄 Pareto front data saved to '{csv_path.name}'")
-
-    if save_format in ['sqlite', 'both']:
-        db_path = save_path / "Optimization_Multi.db"
-        with DatabaseManager(db_path) as db:
-            db.insert_from_dataframe(
-                table_name=sanitized_run_name,
-                df=results_df,
-                if_exists='replace'
-            )
-        _LOGGER.info(f"🗃️ Pareto front data saved to table '{sanitized_run_name}' in '{db_path.name}'")
-
-    # --- Plotting Logic ---
-    plot_pareto_front(
-        results_df,
-        objective_cols=objective_cols,
-        save_path=save_path / f"pareto_plot_{sanitized_run_name}.svg"
-    )
-
-    if logger:
-        log_df = logger.to_dataframe()
-        save_dataframe(df=log_df, save_dir=save_path / "EvolutionLogs", filename=f"log_{sanitized_run_name}")
-
-
-def plot_pareto_front(results_df: pd.DataFrame, objective_cols: List[str], save_path: Path):
-    """
-    Generates and saves a plot of the Pareto front.
-
-    - For 2 objectives, it creates a 2D scatter plot.
-    - For 3 objectives, it creates a 3D scatter plot.
-    - For >3 objectives, it creates a scatter plot matrix (pairs plot).
-
-    Args:
-        results_df (pd.DataFrame): DataFrame containing the optimization results.
-        objective_cols (List[str]): The names of the columns that hold the objective values.
-        save_path (Path): The full path (including filename) to save the SVG plot.
-    """
-    num_objectives = len(objective_cols)
-    _LOGGER.info(f"🎨 Generating Pareto front plot for {num_objectives} objectives...")
-
-    plt.style.use('seaborn-v0_8-whitegrid')
-
-    if num_objectives == 2:
-        fig, ax = plt.subplots(figsize=(8, 6), dpi=120)
-        ax.scatter(results_df[objective_cols[0]], results_df[objective_cols[1]], alpha=0.7, edgecolors='k')
-        ax.set_xlabel(objective_cols[0])
-        ax.set_ylabel(objective_cols[1])
-        ax.set_title("Pareto Front (2D)")
-    
-    elif num_objectives == 3:
-        fig = plt.figure(figsize=(9, 7), dpi=120)
-        ax = fig.add_subplot(111, projection='3d')
-        ax.scatter(results_df[objective_cols[0]], results_df[objective_cols[1]], results_df[objective_cols[2]], alpha=0.7, depthshade=True)
-        ax.set_xlabel(objective_cols[0])
-        ax.set_ylabel(objective_cols[1])
-        ax.set_zlabel(objective_cols[2])
-        ax.set_title("Pareto Front (3D)")
-
-    else: # > 3 objectives
-        _LOGGER.info("  -> More than 3 objectives found, generating a scatter plot matrix.")
-        g = sns.pairplot(results_df[objective_cols], diag_kind="kde", plot_kws={'alpha': 0.6})
-        g.fig.suptitle("Pareto Front (Pairs Plot)", y=1.02)
-        plt.savefig(save_path, bbox_inches='tight')
-        plt.close()
-        _LOGGER.info(f"📊 Pareto plot saved to '{save_path.name}'")
-        return
-
-    plt.tight_layout()
-    plt.savefig(save_path)
-    plt.close()
-    _LOGGER.info(f"📊 Pareto plot saved to '{save_path.name}'")
-