dragon-ml-toolbox 10.1.1__py3-none-any.whl → 14.2.0__py3-none-any.whl

ml_tools/PSO_optimization.py

@@ -4,20 +4,23 @@ import xgboost as xgb
 import lightgbm as lgb
 from typing import Literal, Union, Tuple, Dict, Optional
 from copy import deepcopy
-from .utilities import (
-    threshold_binary_values, 
-    threshold_binary_values_batch, 
-    deserialize_object)
-from .path_manager import sanitize_filename, make_fullpath, list_files_by_extension
 import torch
 from tqdm import trange
+from contextlib import nullcontext
+
+from .serde import deserialize_object
+from .math_utilities import threshold_binary_values, threshold_binary_values_batch
+from .path_manager import sanitize_filename, make_fullpath, list_files_by_extension
 from ._logger import _LOGGER
 from .keys import EnsembleKeys
 from ._script_info import _script_info
 from .SQL import DatabaseManager
-from contextlib import nullcontext
 from .optimization_tools import _save_result
 
+"""
+DEPRECATED
+"""
+
 
 __all__ = [
     "ObjectiveFunction",
@@ -47,7 +50,7 @@ class ObjectiveFunction():
         self.binary_features = binary_features
         self.is_hybrid = False if binary_features <= 0 else True
         self.use_noise = add_noise
-        self._artifact = deserialize_object(trained_model_path, verbose=False, raise_on_error=True)
+        self._artifact = deserialize_object(trained_model_path, verbose=False)
         self.model = self._get_from_artifact(EnsembleKeys.MODEL)
         self.feature_names: Optional[list[str]] = self._get_from_artifact(EnsembleKeys.FEATURES) # type: ignore
         self.target_name: Optional[str] = self._get_from_artifact(EnsembleKeys.TARGET) # type: ignore

ml_tools/RNN_forecast.py

@@ -1,8 +1,10 @@
 import torch
 from torch import nn
 import numpy as np
+
 from ._script_info import _script_info
 
+
 __all__ = [
     "rnn_forecast"
 ]

ml_tools/SQL.py

@@ -2,9 +2,10 @@ import sqlite3
 import pandas as pd
 from pathlib import Path
 from typing import Union, Dict, Any, Optional, List, Literal
+
 from ._logger import _LOGGER
 from ._script_info import _script_info
-from .path_manager import make_fullpath
+from .path_manager import make_fullpath, sanitize_filename
 
 
 __all__ = [
@@ -94,11 +95,13 @@ class DatabaseManager:
         if not self.cursor:
             _LOGGER.error("Database connection is not open.")
             raise sqlite3.Error()
+        
+        sanitized_table_name = sanitize_filename(table_name)
 
         columns_def = ", ".join([f'"{col_name}" {col_type}' for col_name, col_type in schema.items()])
         exists_clause = "IF NOT EXISTS" if if_not_exists else ""
         
-        query = f"CREATE TABLE {exists_clause} {table_name} ({columns_def})"
+        query = f"CREATE TABLE {exists_clause} {sanitized_table_name} ({columns_def})"
         
         _LOGGER.info(f"➡️ Executing: {query}")
         self.cursor.execute(query)
@@ -118,12 +121,14 @@ class DatabaseManager:
         if not self.cursor:
             _LOGGER.error("Database connection is not open.")
             raise sqlite3.Error()
+        
+        sanitized_table_name = sanitize_filename(table_name)
 
         columns = ', '.join(f'"{k}"' for k in data.keys())
         placeholders = ', '.join(['?'] * len(data))
         values = list(data.values())
         
-        query = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
+        query = f'INSERT INTO "{sanitized_table_name}" ({columns}) VALUES ({placeholders})'
         
         self.cursor.execute(query, values)
 
@@ -185,6 +190,8 @@ class DatabaseManager:
         if not data:
             _LOGGER.warning("'insert_many' called with empty data list. No action taken.")
             return
+        
+        sanitized_table_name = sanitize_filename(table_name)
 
         # Assume all dicts have the same keys as the first one
         first_row = data[0]
@@ -194,10 +201,10 @@ class DatabaseManager:
         # Create a list of tuples, where each tuple is a row of values
         values_to_insert = [list(row.values()) for row in data]
 
-        query = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
+        query = f'INSERT INTO "{sanitized_table_name}" ({columns}) VALUES ({placeholders})'
         
         self.cursor.executemany(query, values_to_insert)
-        _LOGGER.info(f"➡️ Bulk inserted {len(values_to_insert)} rows into '{table_name}'.")
+        _LOGGER.info(f"➡️ Bulk inserted {len(values_to_insert)} rows into '{sanitized_table_name}'.")
         
     def insert_from_dataframe(self, table_name: str, df: pd.DataFrame, if_exists: Literal['fail', 'replace', 'append'] = 'append'):
         """
@@ -218,9 +225,11 @@ class DatabaseManager:
         if not self.conn:
             _LOGGER.error("Database connection is not open.")
             raise sqlite3.Error()
+        
+        sanitized_table_name = sanitize_filename(table_name)
 
         df.to_sql(
-            table_name,
+            sanitized_table_name,
             self.conn,
             if_exists=if_exists,
             index=False  # Typically, we don't want to save the DataFrame index
@@ -246,9 +255,11 @@ class DatabaseManager:
         if not self.conn:
             _LOGGER.error("Database connection is not open.")
             raise sqlite3.Error()
+        
+        sanitized_table_name = sanitize_filename(table_name)
             
         # PRAGMA is a special SQL command in SQLite for database metadata
-        return pd.read_sql_query(f'PRAGMA table_info("{table_name}");', self.conn)
+        return pd.read_sql_query(f'PRAGMA table_info("{sanitized_table_name}");', self.conn)
 
     def create_index(self, table_name: str, column_name: str, unique: bool = False):
         """
@@ -267,11 +278,13 @@ class DatabaseManager:
         if not self.cursor:
             _LOGGER.error("Database connection is not open.")
             raise sqlite3.Error()
+        
+        sanitized_table_name = sanitize_filename(table_name)
 
-        index_name = f"idx_{table_name}_{column_name}"
+        index_name = f"idx_{sanitized_table_name}_{column_name}"
         unique_clause = "UNIQUE" if unique else ""
         
-        query = f"CREATE {unique_clause} INDEX IF NOT EXISTS {index_name} ON {table_name} ({column_name})"
+        query = f'CREATE {unique_clause} INDEX IF NOT EXISTS "{index_name}" ON "{sanitized_table_name}" ("{column_name}")'
         
         _LOGGER.info(f"➡️ Executing: {query}")
         self.cursor.execute(query)

ml_tools/VIF_factor.py

@@ -1,4 +1,3 @@
-
 import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
@@ -7,11 +6,13 @@ from statsmodels.stats.outliers_influence import variance_inflation_factor
 from statsmodels.tools.tools import add_constant
 import warnings
 from pathlib import Path
-from .utilities import yield_dataframes_from_dir, save_dataframe
+
+from .utilities import yield_dataframes_from_dir, save_dataframe_filename
 from .path_manager import sanitize_filename, make_fullpath
 from ._logger import _LOGGER
 from ._script_info import _script_info
 
+
 __all__ = [
     "compute_vif",
     "drop_vif_based",
@@ -228,7 +229,7 @@ def compute_vif_multi(input_directory: Union[str, Path],
             result_df, dropped_cols = drop_vif_based(df=df, vif_df=vif_dataframe)
             
             if len(dropped_cols) > 0:
-                save_dataframe(df=result_df, save_dir=output_dataset_path, filename=new_filename)
+                save_dataframe_filename(df=result_df, save_dir=output_dataset_path, filename=new_filename)
 
 
 def info():

ml_tools/_ML_vision_recipe.py

@@ -0,0 +1,88 @@
+import json
+import torch
+from torchvision import transforms
+from typing import Dict, Any, List, Callable, Union
+from pathlib import Path
+
+from .ML_vision_transformers import TRANSFORM_REGISTRY
+from ._logger import _LOGGER
+from .keys import VisionTransformRecipeKeys
+from .path_manager import make_fullpath
+
+
+def save_recipe(recipe: Dict[str, Any], filepath: Path) -> None:
+    """
+    Saves a transform recipe dictionary to a JSON file.
+
+    Args:
+        recipe (Dict[str, Any]): The recipe dictionary to save.
+        filepath (str): The path to the output .json file.
+    """
+    final_filepath = filepath.with_suffix(".json")
+    
+    try:
+        with open(final_filepath, 'w') as f:
+            json.dump(recipe, f, indent=4)
+        _LOGGER.info(f"Transform recipe saved as '{final_filepath.name}'.")
+    except Exception as e:
+        _LOGGER.error(f"Failed to save recipe to '{final_filepath}': {e}")
+        raise
+
+
+def load_recipe_and_build_transform(filepath: Union[str,Path]) -> transforms.Compose:
+    """
+    Loads a transform recipe from a .json file and reconstructs the
+    torchvision.transforms.Compose pipeline.
+
+    Args:
+        filepath (str): Path to the saved transform recipe .json file.
+
+    Returns:
+        transforms.Compose: The reconstructed transformation pipeline.
+        
+    Raises:
+        ValueError: If a transform name in the recipe is not found in
+                    torchvision.transforms or the custom TRANSFORM_REGISTRY.
+    """
+    # validate filepath
+    final_filepath = make_fullpath(filepath, enforce="file")
+    
+    try:
+        with open(final_filepath, 'r') as f:
+            recipe = json.load(f)
+    except Exception as e:
+        _LOGGER.error(f"Failed to load recipe from '{final_filepath}': {e}")
+        raise
+        
+    pipeline_steps: List[Callable] = []
+    
+    if VisionTransformRecipeKeys.PIPELINE not in recipe:
+        _LOGGER.error("Recipe file is invalid: missing 'pipeline' key.")
+        raise ValueError("Invalid recipe format.")
+
+    for step in recipe[VisionTransformRecipeKeys.PIPELINE]:
+        t_name = step[VisionTransformRecipeKeys.NAME]
+        t_kwargs = step[VisionTransformRecipeKeys.KWARGS]
+        
+        transform_class: Any = None
+
+        # 1. Check standard torchvision transforms
+        if hasattr(transforms, t_name):
+            transform_class = getattr(transforms, t_name)
+        # 2. Check custom transforms
+        elif t_name in TRANSFORM_REGISTRY:
+            transform_class = TRANSFORM_REGISTRY[t_name]
+        # 3. Not found
+        else:
+            _LOGGER.error(f"Unknown transform '{t_name}' in recipe. Not found in torchvision.transforms or TRANSFORM_REGISTRY.")
+            raise ValueError(f"Unknown transform name: {t_name}")
+            
+        # Instantiate the transform
+        try:
+            pipeline_steps.append(transform_class(**t_kwargs))
+        except Exception as e:
+            _LOGGER.error(f"Failed to instantiate transform '{t_name}' with kwargs {t_kwargs}: {e}")
+            raise
+            
+    _LOGGER.info(f"Successfully loaded and built transform pipeline from '{final_filepath.name}'.")
+    return transforms.Compose(pipeline_steps)

ml_tools/__init__.py

@@ -1 +1,2 @@
 from .custom_logger import custom_logger
+from ._schema import FeatureSchema

ml_tools/_logger.py

@@ -8,8 +8,6 @@ except ImportError:
     colorlog = None
 
 
-
-
 # --- Centralized Configuration ---
 LEVEL_EMOJIS = {
     logging.INFO: "✅",

ml_tools/_schema.py

@@ -0,0 +1,96 @@
+from typing import NamedTuple, Tuple, Optional, Dict, Union
+from pathlib import Path
+
+from .custom_logger import save_list_strings
+from .keys import DatasetKeys
+from ._logger import _LOGGER
+
+
+class FeatureSchema(NamedTuple):
+    """Holds the final, definitive schema for the model pipeline."""
+    
+    # The final, ordered list of all feature names
+    feature_names: Tuple[str, ...]
+    
+    # List of all continuous feature names
+    continuous_feature_names: Tuple[str, ...]
+    
+    # List of all categorical feature names
+    categorical_feature_names: Tuple[str, ...]
+    
+    # Map of {column_index: cardinality} for categorical features
+    categorical_index_map: Optional[Dict[int, int]]
+    
+    # Map string-to-int category values (e.g., {'color': {'red': 0, 'blue': 1}})
+    categorical_mappings: Optional[Dict[str, Dict[str, int]]]
+
+    def _save_helper(self, artifact: Tuple[str, ...], directory: Union[str,Path], filename: str, verbose: bool):
+        to_save = list(artifact)
+        
+        # empty check
+        if not to_save:
+            _LOGGER.warning(f"Skipping save for '{filename}': The feature list is empty.")
+            return
+        
+        save_list_strings(list_strings=to_save,
+                          directory=directory,
+                          filename=filename,
+                          verbose=verbose)
+
+    def save_all_features(self, directory: Union[str,Path], verbose: bool=True):
+        """
+        Saves all feature names to a text file.
+
+        Args:
+            directory: The directory where the file will be saved.
+            verbose: If True, prints a confirmation message upon saving.
+        """
+        self._save_helper(artifact=self.feature_names,
+                          directory=directory,
+                          filename=DatasetKeys.FEATURE_NAMES,
+                          verbose=verbose)
+        
+    def save_continuous_features(self, directory: Union[str,Path], verbose: bool=True):
+        """
+        Saves continuous feature names to a text file.
+
+        Args:
+            directory: The directory where the file will be saved.
+            verbose: If True, prints a confirmation message upon saving.
+        """
+        self._save_helper(artifact=self.continuous_feature_names,
+                          directory=directory,
+                          filename=DatasetKeys.CONTINUOUS_NAMES,
+                          verbose=verbose)
+    
+    def save_categorical_features(self, directory: Union[str,Path], verbose: bool=True):
+        """
+        Saves categorical feature names to a text file.
+
+        Args:
+            directory: The directory where the file will be saved.
+            verbose: If True, prints a confirmation message upon saving.
+        """
+        self._save_helper(artifact=self.categorical_feature_names,
+                          directory=directory,
+                          filename=DatasetKeys.CATEGORICAL_NAMES,
+                          verbose=verbose)
+        
+    def save_artifacts(self, directory: Union[str,Path]):
+        """
+        Saves feature names, categorical feature names, continuous feature names to separate text files.
+        """
+        self.save_all_features(directory=directory, verbose=True)
+        self.save_continuous_features(directory=directory, verbose=True)
+        self.save_categorical_features(directory=directory, verbose=True)
+        
+    def __repr__(self) -> str:
+        """Returns a concise representation of the schema's contents."""
+        total = len(self.feature_names)
+        cont = len(self.continuous_feature_names)
+        cat = len(self.categorical_feature_names)
+        index_map = self.categorical_index_map is not None
+        cat_map = self.categorical_mappings is not None
+        return (
+            f"<FeatureSchema(total={total}, continuous={cont}, categorical={cat}, index_map={index_map}, categorical_map={cat_map})>"
+        )

ml_tools/constants.py

@@ -0,0 +1,79 @@
+CHEMICAL_ELEMENTS = [
+    "Hydrogen", "Helium", "Lithium", "Beryllium", "Boron", "Carbon", "Nitrogen", "Oxygen", "Fluorine", "Neon",
+    "Sodium", "Magnesium", "Aluminum", "Silicon", "Phosphorus", "Sulfur", "Chlorine", "Argon",
+    "Potassium", "Calcium", "Scandium", "Titanium", "Vanadium", "Chromium", "Manganese", "Iron", "Cobalt", "Nickel", "Copper", "Zinc",
+    "Gallium", "Germanium", "Arsenic", "Selenium", "Bromine", "Krypton",
+    "Rubidium", "Strontium", "Yttrium", "Zirconium", "Niobium", "Molybdenum", "Technetium", "Ruthenium", "Rhodium", "Palladium", "Silver", "Cadmium",
+    "Indium", "Tin", "Antimony", "Tellurium", "Iodine", "Xenon",
+    "Cesium", "Barium", "Lanthanum", "Cerium", "Praseodymium", "Neodymium", "Promethium", "Samarium", "Europium", "Gadolinium", "Terbium", "Dysprosium", "Holmium", "Erbium", "Thulium", "Ytterbium", "Lutetium",
+    "Hafnium", "Tantalum", "Tungsten", "Rhenium", "Osmium", "Iridium", "Platinum", "Gold", "Mercury",
+    "Thallium", "Lead", "Bismuth", "Polonium", "Astatine", "Radon",
+    "Francium", "Radium", "Actinium", "Thorium", "Protactinium", "Uranium", "Neptunium", "Plutonium", "Americium", "Curium", "Berkelium", "Californium", "Einsteinium", "Fermium", "Mendelevium", "Nobelium", "Lawrencium",
+    "Rutherfordium", "Dubnium", "Seaborgium", "Bohrium", "Hassium", "Meitnerium", "Darmstadtium", "Roentgenium", "Copernicium", "Nihonium", "Flerovium", "Moscovium", "Livermorium", "Tennessine", "Oganesson"
+]
+
+CHEMICAL_ELEMENT_SYMBOLS = [
+    "H", "He", "Li", "Be", "B", "C", "N", "O", "F", "Ne",
+    "Na", "Mg", "Al", "Si", "P", "S", "Cl", "Ar",
+    "K", "Ca", "Sc", "Ti", "V", "Cr", "Mn", "Fe", "Co", "Ni", "Cu", "Zn",
+    "Ga", "Ge", "As", "Se", "Br", "Kr",
+    "Rb", "Sr", "Y", "Zr", "Nb", "Mo", "Tc", "Ru", "Rh", "Pd", "Ag", "Cd",
+    "In", "Sn", "Sb", "Te", "I", "Xe",
+    "Cs", "Ba", "La", "Ce", "Pr", "Nd", "Pm", "Sm", "Eu", "Gd", "Tb", "Dy", "Ho", "Er", "Tm", "Yb", "Lu",
+    "Hf", "Ta", "W", "Re", "Os", "Ir", "Pt", "Au", "Hg",
+    "Tl", "Pb", "Bi", "Po", "At", "Rn",
+    "Fr", "Ra", "Ac", "Th", "Pa", "U", "Np", "Pu", "Am", "Cm", "Bk", "Cf", "Es", "Fm", "Md", "No", "Lr",
+    "Rf", "Db", "Sg", "Bh", "Hs", "Mt", "Ds", "Rg", "Cn", "Nh", "Fl", "Mc", "Lv", "Ts", "Og"
+]
+
+# --- Physics & Chemistry ---
+
+# Speed of light in vacuum (m/s)
+SPEED_OF_LIGHT = 299792458.0
+
+# Planck constant (J·s)
+PLANCK_CONSTANT = 6.62607015e-34
+
+# Avogadro's number (mol⁻¹)
+AVOGADRO_NUMBER = 6.02214076e23
+
+# Universal gas constant (J/(mol·K))
+UNIVERSAL_GAS_CONSTANT = 8.314462618
+
+# Boltzmann constant (J/K)
+BOLTZMANN_CONSTANT = 1.380649e-23
+
+# Gravitational constant (m³·kg⁻¹·s⁻²)
+GRAVITATIONAL_CONSTANT = 6.67430e-11
+
+# Standard acceleration of gravity on Earth (m/s²)
+STANDARD_GRAVITY = 9.80665
+
+# Elementary charge (C)
+ELEMENTARY_CHARGE = 1.602176634e-19
+
+# Electron mass (kg)
+ELECTRON_MASS_KG = 9.1093837015e-31
+
+# Proton mass (kg)
+PROTON_MASS_KG = 1.67262192369e-27
+
+# Absolute zero (in Celsius)
+ABSOLUTE_ZERO_CELSIUS = -273.15
+
+# --- Astronomy ---
+
+# Astronomical Unit, the mean Earth-Sun distance (meters)
+ASTRONOMICAL_UNIT_KM = 149597870.7
+
+# Light-year (meters)
+LIGHT_YEAR_KM = 9460730472580.8
+
+# Earth's equatorial radius (meters)
+EARTH_RADIUS_KM = 6378.137
+
+# Mass of the Earth (kg)
+EARTH_MASS_KG = 5.9722e24
+
+# Mass of the Sun (kg)
+SUN_MASS_KG = 1.98847e30