dragon-ml-toolbox 6.0.1__py3-none-any.whl → 6.1.1__py3-none-any.whl

{dragon_ml_toolbox-6.0.1.dist-info → dragon_ml_toolbox-6.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 6.0.1
+Version: 6.1.1
 Summary: A collection of tools for data science and machine learning projects.
 Author-email: Karl Loza <luigiloza@gmail.com>
 License-Expression: MIT

{dragon_ml_toolbox-6.0.1.dist-info → dragon_ml_toolbox-6.1.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
-dragon_ml_toolbox-6.0.1.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
-dragon_ml_toolbox-6.0.1.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=lY4_rJPnLnMu7YBQaY-_iz1JRDcLdQzNCyeLAF1glJY,1837
+dragon_ml_toolbox-6.1.1.dist-info/licenses/LICENSE,sha256=2uUFNy7D0TLgHim1K5s3DIJ4q_KvxEXVilnU20cWliY,1066
+dragon_ml_toolbox-6.1.1.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=lY4_rJPnLnMu7YBQaY-_iz1JRDcLdQzNCyeLAF1glJY,1837
 ml_tools/ETL_engineering.py,sha256=4wwZXi9_U7xfCY70jGBaKniOeZ0m75ppxWpQBd_DmLc,39369
 ml_tools/GUI_tools.py,sha256=n4ZZ5kEjwK5rkOCFJE41HeLFfjhpJVLUSzk9Kd9Kr_0,45410
 ml_tools/MICE_imputation.py,sha256=oFHg-OytOzPYTzBR_wIRHhP71cMn3aupDeT59ABsXlQ,11576
 ml_tools/ML_callbacks.py,sha256=FEJ80TSEtY0-hdnOsAWeVApQt1mdzTdOntqtoWmMAzE,13310
 ml_tools/ML_datasetmaster.py,sha256=bbKCNA_b_uDIfxP9YIYKZm-VSfUSD15LvegFxpE9DIQ,34315
 ml_tools/ML_evaluation.py,sha256=-Z5fXQi2ou6l5Oyir06bO90SZIZVrjQfgoVAqKgSjks,13800
-ml_tools/ML_inference.py,sha256=Fh-X2UQn3AznWBjf-7iPSxwE-EzkGQm1VEIRUAkURmE,5336
+ml_tools/ML_inference.py,sha256=blEDgzvDqatxbfloBKsyNPacRwoq9g6WTpIKQ3zoTak,5758
 ml_tools/ML_models.py,sha256=SJhKHGAN2VTBqzcHUOpFWuVZ2Y7U1M4P_axG_LNYWcI,6460
-ml_tools/ML_optimization.py,sha256=zGKpWW4SL1-3iiHglDP-dkuADL73T0kxs3Dc-Lyishs,9671
+ml_tools/ML_optimization.py,sha256=0kRkjcAHbbx6EINUjzKfibL5h0DV39wghjcjzN0syNI,13406
 ml_tools/ML_trainer.py,sha256=1q_CDXuMfndRsPuNofUn2mg2TlhG6MYuGqjWxTDgN9c,15112
 ml_tools/PSO_optimization.py,sha256=9Y074d-B5h4Wvp9YPiy6KAeXM-Yv6Il3gWalKvOLVgo,22705
 ml_tools/RNN_forecast.py,sha256=2CyjBLSYYc3xLHxwLXUmP5Qv8AmV1OB_EndETNX1IBk,1956
@@ -27,7 +27,7 @@ ml_tools/keys.py,sha256=HtPG8-MWh89C32A7eIlfuuA-DLwkxGkoDfwR2TGN9CQ,1074
 ml_tools/optimization_tools.py,sha256=MuT4OG7_r1QqLUti-yYix7QeCpglezD0oe9BDCq0QXk,5086
 ml_tools/path_manager.py,sha256=Z8e7w3MPqQaN8xmTnKuXZS6CIW59BFwwqGhGc00sdp4,13692
 ml_tools/utilities.py,sha256=LqXXTovaHbA5AOKRk6Ru6DgAPAM0wPfYU70kUjYBryo,19231
-dragon_ml_toolbox-6.0.1.dist-info/METADATA,sha256=SxZPqt9cAVNkerRZYCpZP_-v7feEx5MTK5lCXeA5dxc,6698
-dragon_ml_toolbox-6.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dragon_ml_toolbox-6.0.1.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
-dragon_ml_toolbox-6.0.1.dist-info/RECORD,,
+dragon_ml_toolbox-6.1.1.dist-info/METADATA,sha256=qrfNT_c9zH8iYfbe_QBoxpRJNLvrhm-ZKQyMeVwDu9w,6698
+dragon_ml_toolbox-6.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dragon_ml_toolbox-6.1.1.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
+dragon_ml_toolbox-6.1.1.dist-info/RECORD,,

ml_tools/ML_inference.py

@@ -66,47 +66,10 @@ class PyTorchInferenceHandler:
         
         # Ensure tensor is on the correct device
         return features.to(self.device)
-
-    def predict(self, features: Union[np.ndarray, torch.Tensor]) -> Dict[str, Any]:
-        """
-        Predicts on a single feature vector.
-
-        Args:
-            features (np.ndarray | torch.Tensor): A 1D or 2D array/tensor for a single sample.
-
-        Returns:
-            Dict[str, Any]: A dictionary containing the prediction.
-                - For regression: {'predictions': float}
-                - For classification: {'labels': int, 'probabilities': np.ndarray}
+    
+    def predict_batch(self, features: Union[np.ndarray, torch.Tensor]) -> Dict[str, torch.Tensor]:
         """
-        if features.ndim == 1:
-            features = features.reshape(1, -1)
-        
-        if features.shape[0] != 1:
-            raise ValueError("The predict() method is for a single sample. Use predict_batch() for multiple samples.")
-
-        results_batch = self.predict_batch(features)
-
-        # Extract the single result from the batch
-        if self.task == "regression":
-            return {PyTorchInferenceKeys.PREDICTIONS: results_batch[PyTorchInferenceKeys.PREDICTIONS].item()}
-        else: # classification
-            return {
-                PyTorchInferenceKeys.LABELS: results_batch[PyTorchInferenceKeys.LABELS].item(),
-                PyTorchInferenceKeys.PROBABILITIES: results_batch[PyTorchInferenceKeys.PROBABILITIES][0]
-            }
-
-    def predict_batch(self, features: Union[np.ndarray, torch.Tensor]) -> Dict[str, Any]:
-        """
-        Predicts on a batch of feature vectors.
-
-        Args:
-            features (np.ndarray | torch.Tensor): A 2D array/tensor where each row is a sample.
-
-        Returns:
-            Dict[str, Any]: A dictionary containing the predictions.
-                - For regression: {'predictions': np.ndarray}
-                - For classification: {'labels': np.ndarray, 'probabilities': np.ndarray}
+        Core batch prediction method. Returns results as PyTorch tensors on the model's device.
         """
         if features.ndim != 2:
             raise ValueError("Input for batch prediction must be a 2D array or tensor.")
@@ -114,18 +77,61 @@ class PyTorchInferenceHandler:
         input_tensor = self._preprocess_input(features)
         
         with torch.no_grad():
-            output = self.model(input_tensor).cpu()
+            # Output tensor remains on the model's device (e.g., 'mps' or 'cuda')
+            output = self.model(input_tensor)
 
             if self.task == "classification":
                 probs = nn.functional.softmax(output, dim=1)
                 labels = torch.argmax(probs, dim=1)
                 return {
-                    PyTorchInferenceKeys.LABELS: labels.numpy(),
-                    PyTorchInferenceKeys.PROBABILITIES: probs.numpy()
+                    PyTorchInferenceKeys.LABELS: labels,
+                    PyTorchInferenceKeys.PROBABILITIES: probs
                 }
             else:  # regression
-                return {PyTorchInferenceKeys.PREDICTIONS: output.numpy()}
+                return {PyTorchInferenceKeys.PREDICTIONS: output}
 
+    def predict(self, features: Union[np.ndarray, torch.Tensor]) -> Dict[str, torch.Tensor]:
+        """
+        Core single-sample prediction. Returns results as PyTorch tensors on the model's device.
+        """
+        if features.ndim == 1:
+            features = features.reshape(1, -1)
+        
+        if features.shape[0] != 1:
+            raise ValueError("The predict() method is for a single sample. Use predict_batch() for multiple samples.")
+
+        batch_results = self.predict_batch(features)
+        
+        single_results = {key: value[0] for key, value in batch_results.items()}
+        return single_results
+
+    # --- NumPy Convenience Wrappers (on CPU) ---
+
+    def predict_batch_numpy(self, features: Union[np.ndarray, torch.Tensor]) -> Dict[str, np.ndarray]:
+        """
+        Convenience wrapper for predict_batch that returns NumPy arrays.
+        """
+        tensor_results = self.predict_batch(features)
+        # Move tensor to CPU before converting to NumPy
+        numpy_results = {key: value.cpu().numpy() for key, value in tensor_results.items()}
+        return numpy_results
+
+    def predict_numpy(self, features: Union[np.ndarray, torch.Tensor]) -> Dict[str, Any]:
+        """
+        Convenience wrapper for predict that returns NumPy arrays or scalars.
+        """
+        tensor_results = self.predict(features)
+        
+        if self.task == "regression":
+            # .item() implicitly moves to CPU
+            return {PyTorchInferenceKeys.PREDICTIONS: tensor_results[PyTorchInferenceKeys.PREDICTIONS].item()}
+        else: # classification
+            return {
+                PyTorchInferenceKeys.LABELS: tensor_results[PyTorchInferenceKeys.LABELS].item(),
+                # ✅ Move tensor to CPU before converting to NumPy
+                PyTorchInferenceKeys.PROBABILITIES: tensor_results[PyTorchInferenceKeys.PROBABILITIES].cpu().numpy()
+            }
+     
 
 def info():
     _script_info(__all__)

ml_tools/ML_optimization.py

@@ -1,12 +1,15 @@
+import pandas # logger
 import torch
 import numpy    #handling torch to numpy
 import evotorch
-from evotorch.algorithms import CMAES, SteadyStateGA
-from evotorch.logging import StdOutLogger
-from typing import Literal, Union, Tuple, List, Optional
+from evotorch.algorithms import SNES, CEM, GeneticAlgorithm
+from evotorch.logging import PandasLogger
+from evotorch.operators import SimulatedBinaryCrossOver, GaussianMutation
+from typing import Literal, Union, Tuple, List, Optional, Any, Callable
 from pathlib import Path
 from tqdm.auto import trange
 from contextlib import nullcontext
+from functools import partial
 
 from .path_manager import make_fullpath, sanitize_filename
 from ._logger import _LOGGER
@@ -15,8 +18,7 @@ from .ML_inference import PyTorchInferenceHandler
 from .keys import PyTorchInferenceKeys
 from .SQL import DatabaseManager
 from .optimization_tools import _save_result
-from .utilities import threshold_binary_values
-
+from .utilities import threshold_binary_values, save_dataframe
 
 __all__ = [
     "create_pytorch_problem",
@@ -25,34 +27,38 @@ __all__ = [
 
 
 def create_pytorch_problem(
-    handler: PyTorchInferenceHandler,
+    inference_handler: PyTorchInferenceHandler,
     bounds: Tuple[List[float], List[float]],
     binary_features: int,
-    task: Literal["minimize", "maximize"],
-    algorithm: Literal["CMAES", "GA"] = "CMAES",
-    verbose: bool = False,
+    task: Literal["min", "max"],
+    algorithm: Literal["SNES", "CEM", "Genetic"] = "Genetic",
+    population_size: int = 200,
     **searcher_kwargs
-) -> Tuple[evotorch.Problem, evotorch.Searcher]: # type: ignore
+) -> Tuple[evotorch.Problem, Callable[[], Any]]:
     """
-    Creates and configures an EvoTorch Problem and Searcher for a PyTorch model.
-
+    Creates and configures an EvoTorch Problem and a Searcher factory class for a PyTorch model.
+    
+    SNES and CEM do not accept bounds, the given bounds will be used as initial bounds only.
+    
+    The Genetic Algorithm works directly with the bounds, and operators such as SimulatedBinaryCrossOver and GaussianMutation.
+    
     Args:
-        handler (PyTorchInferenceHandler): An initialized inference handler
-            containing the model and weights.
-        bounds (tuple[list[float], list[float]]): A tuple containing the lower
-            and upper bounds for the solution features.
+        inference_handler (PyTorchInferenceHandler): An initialized inference handler containing the model and weights.
+        bounds (tuple[list[float], list[float]]): A tuple containing the lower and upper bounds for the solution features.
         binary_features (int): Number of binary features located at the END of the feature vector. Will be automatically added to the bounds.
         task (str): The optimization goal, either "minimize" or "maximize".
-        algorithm (str): The search algorithm to use, "CMAES" or "GA" (SteadyStateGA).
-        verbose (bool): Add an Evotorch logger for real-time console updates.
+        algorithm (str): The search algorithm to use.
+        population_size (int): Used for CEM and GeneticAlgorithm.
         **searcher_kwargs: Additional keyword arguments to pass to the
             selected search algorithm's constructor (e.g., stdev_init=0.5 for CMAES).
 
     Returns:
         Tuple:
-        A tuple containing the configured evotorch.Problem and evotorch.Searcher.
+        A tuple containing the configured Problem and Searcher.
     """
-    lower_bounds, upper_bounds = bounds
+    # Create copies to avoid modifying the original lists passed in the `bounds` tuple
+    lower_bounds = list(bounds[0])
+    upper_bounds = list(bounds[1])
     
     # add binary bounds
     if binary_features > 0:
@@ -60,51 +66,86 @@ def create_pytorch_problem(
         upper_bounds.extend([0.55] * binary_features)
     
     solution_length = len(lower_bounds)
-    device = handler.device
+    device = inference_handler.device
 
     # Define the fitness function that EvoTorch will call.
-    @evotorch.decorators.to_tensor # type: ignore
-    @evotorch.decorators.on_aux_device(device)
     def fitness_func(solution_tensor: torch.Tensor) -> torch.Tensor:
         # Directly use the continuous-valued tensor from the optimizer for prediction
-        predictions = handler.predict_batch(solution_tensor)[PyTorchInferenceKeys.PREDICTIONS]
+        predictions = inference_handler.predict_batch(solution_tensor)[PyTorchInferenceKeys.PREDICTIONS]
         return predictions.flatten()
-
+    
+    
     # Create the Problem instance.
-    problem = evotorch.Problem(
-        objective_sense=task,
-        objective_func=fitness_func,
-        solution_length=solution_length,
-        initial_bounds=(lower_bounds, upper_bounds),
-        device=device,
-    )
-
-    # Create the selected searcher instance.
-    if algorithm == "CMAES":
-        searcher = CMAES(problem, **searcher_kwargs)
-    elif algorithm == "GA":
-        searcher = SteadyStateGA(problem, **searcher_kwargs)
-    else:
-        raise ValueError(f"Unknown algorithm '{algorithm}'. Choose 'CMAES' or 'GA'.")
+    if algorithm == "CEM" or algorithm == "SNES":
+        problem = evotorch.Problem(
+            objective_sense=task,
+            objective_func=fitness_func,
+            solution_length=solution_length,
+            initial_bounds=(lower_bounds, upper_bounds),
+            device=device,
+            vectorized=True #Use batches
+        )
+        
+        # If stdev_init is not provided, calculate it based on the bounds (used for SNES and CEM)
+        if 'stdev_init' not in searcher_kwargs:
+            # Calculate stdev for each parameter as 25% of its search range
+            stdevs = [abs(up - low) * 0.25 for low, up in zip(lower_bounds, upper_bounds)]
+            searcher_kwargs['stdev_init'] = torch.tensor(stdevs, dtype=torch.float32, requires_grad=False)
+        
+        if algorithm == "SNES":
+            SearcherClass = SNES
+        elif algorithm == "CEM":
+            SearcherClass = CEM
+            # Set a defaults for CEM if not provided
+            if 'popsize' not in searcher_kwargs:
+                searcher_kwargs['popsize'] = population_size
+            if 'parenthood_ratio' not in searcher_kwargs:
+                searcher_kwargs['parenthood_ratio'] = 0.2   #float 0.0 - 1.0
+        
+    elif algorithm == "Genetic":
+        problem = evotorch.Problem(
+            objective_sense=task,
+            objective_func=fitness_func,
+            solution_length=solution_length,
+            bounds=(lower_bounds, upper_bounds),
+            device=device,
+            vectorized=True #Use batches
+        )
 
-    # Add a logger for real-time console updates.
-    # This gives the user immediate feedback on the optimization progress.
-    if verbose:
-        _ = StdOutLogger(searcher)
+        operators = [
+            SimulatedBinaryCrossOver(problem,
+                                    tournament_size=4,
+                                    eta=0.8),
+            GaussianMutation(problem,
+                            stdev=0.1)
+        ]
+        
+        searcher_kwargs["operators"] = operators
+        if 'popsize' not in searcher_kwargs:
+            searcher_kwargs['popsize'] = population_size
+        
+        SearcherClass = GeneticAlgorithm
+        
+    else:
+        raise ValueError(f"Unknown algorithm '{algorithm}'.")
+    
+    # Create a factory function with all arguments pre-filled
+    searcher_factory = partial(SearcherClass, problem, **searcher_kwargs)
 
-    return problem, searcher
+    return problem, searcher_factory
 
 
 def run_optimization(
     problem: evotorch.Problem,
-    searcher: evotorch.Searcher, # type: ignore
+    searcher_factory: Callable[[],Any],
     num_generations: int,
     target_name: str,
     binary_features: int,
     save_dir: Union[str, Path],
     save_format: Literal['csv', 'sqlite', 'both'],
     feature_names: Optional[List[str]],
-    repetitions: int = 1
+    repetitions: int = 1,
+    verbose: bool = True
 ) -> Optional[dict]:
     """
     Runs the evolutionary optimization process, with support for multiple repetitions.
@@ -124,20 +165,19 @@ def run_optimization(
     Args:
         problem (evotorch.Problem): The configured problem instance, which defines
             the objective function, solution space, and optimization sense.
-        searcher (evotorch.Searcher): The configured searcher instance, which
-            contains the evolutionary algorithm (e.g., CMAES, GA).
-        num_generations (int): The total number of generations to run the
-            search algorithm for in each repetition.
+        searcher_factory (Callable): The searcher factory to generate fresh evolutionary algorithms.
+        num_generations (int): The total number of generations to run the search algorithm for in each repetition.
         target_name (str): Target name that will also be used for the CSV filename and SQL table.
         binary_features (int): Number of binary features located at the END of the feature vector.
         save_dir (str | Path): The directory where the result file(s) will be saved.
         save_format (Literal['csv', 'sqlite', 'both'], optional): The format for
-            saving results during iterative analysis. Defaults to 'both'.
+            saving results during iterative analysis.
         feature_names (List[str], optional): Names of the solution features for
             labeling the output files. If None, generic names like 'feature_0',
-            'feature_1', etc., will be created. Defaults to None.
+            'feature_1', etc., will be created.
         repetitions (int, optional): The number of independent times to run the
-            entire optimization process. Defaults to 1.
+            entire optimization process.
+        verbose (bool): Add an Evotorch Pandas logger saved as a csv. Only for the first repetition.
 
     Returns:
         Optional[dict]: A dictionary containing the best feature values and the
@@ -162,11 +202,29 @@ def run_optimization(
     
     # --- SINGLE RUN LOGIC ---
     if repetitions <= 1:
-        _LOGGER.info(f"🤖 Starting optimization with {searcher.__class__.__name__} for {num_generations} generations...")
-        for _ in trange(num_generations, desc="Optimizing"):
-            searcher.step()
+        searcher = searcher_factory()
+        _LOGGER.info(f"🤖 Starting optimization with {searcher.__class__.__name__} Algorithm for {num_generations} generations...")
+        # for _ in trange(num_generations, desc="Optimizing"):
+        #     searcher.step()
+        
+        # Attach logger if requested
+        if verbose:
+            pandas_logger = PandasLogger(searcher)
+            
+        searcher.run(num_generations) # Use the built-in run method for simplicity
+            
+        # # DEBUG new searcher objects
+        # for status_key in searcher.iter_status_keys():
+        #     print("===", status_key, "===")
+        #     print(searcher.status[status_key])
+        #     print()
+        
+        # Get results from the .status dictionary 
+        # SNES and CEM use the key 'center' to get mean values if needed    best_solution_tensor = searcher.status["center"]
+        best_solution_container = searcher.status["pop_best"]
+        best_solution_tensor = best_solution_container.values
+        best_fitness = best_solution_container.evals
 
-        best_solution_tensor, best_fitness = searcher.best
         best_solution_np = best_solution_tensor.cpu().numpy()
         
         # threshold binary features
@@ -179,6 +237,11 @@ def run_optimization(
         result_dict[target_name] = best_fitness.item()
         
         _save_result(result_dict, 'csv', csv_path) # Single run defaults to CSV
+        
+        # Process logger
+        if verbose:
+            _handle_pandas_log(pandas_logger, save_path=save_path)
+        
         _LOGGER.info(f"✅ Optimization complete. Best solution saved to '{csv_path.name}'")
         return result_dict
 
@@ -193,17 +256,26 @@ def run_optimization(
                 schema = {name: "REAL" for name in feature_names}
                 schema[target_name] = "REAL"
                 db_manager.create_table(db_table_name, schema)
-
+            
+            print("")
+            # Repetitions loop
+            pandas_logger = None
             for i in trange(repetitions, desc="Repetitions"):
-                _LOGGER.info(f"--- Starting Repetition {i+1}/{repetitions} ---")
+                # CRITICAL: Create a fresh searcher for each run using the factory
+                searcher = searcher_factory()
                 
-                # CRITICAL: Re-initialize the searcher to ensure each run is independent
-                searcher.reset()
-
-                for _ in range(num_generations): # Inner loop does not need a progress bar
-                    searcher.step()
-
-                best_solution_tensor, best_fitness = searcher.best
+                # Attach logger if requested
+                if verbose and i==0:
+                    pandas_logger = PandasLogger(searcher)
+                
+                searcher.run(num_generations) # Use the built-in run method for simplicity
+                
+                # Get results from the .status dictionary 
+                # SNES and CEM use the key 'center' to get mean values if needed    best_solution_tensor = searcher.status["center"]
+                best_solution_container = searcher.status["pop_best"]
+                best_solution_tensor = best_solution_container.values
+                best_fitness = best_solution_container.evals
+             
                 best_solution_np = best_solution_tensor.cpu().numpy()
                 
                 # threshold binary features
@@ -212,15 +284,25 @@ def run_optimization(
                 else:
                     best_solution_thresholded = best_solution_np
                 
+                # make results dictionary
                 result_dict = {name: value for name, value in zip(feature_names, best_solution_thresholded)}
                 result_dict[target_name] = best_fitness.item()
                 
                 # Save each result incrementally
                 _save_result(result_dict, save_format, csv_path, db_manager, db_table_name)
+            
+        # Process logger
+        if pandas_logger is not None:
+            _handle_pandas_log(pandas_logger, save_path=save_path)      
         
         _LOGGER.info(f"✅ Optimal solution space complete. Results saved to '{save_path}'")
         return None
 
 
+def _handle_pandas_log(logger: PandasLogger, save_path: Path):
+    log_dataframe = logger.to_dataframe()
+    save_dataframe(df=log_dataframe, save_dir=save_path / "EvolutionLog", filename="evolution")
+
+
 def info():
     _script_info(__all__)