dragon-ml-toolbox 12.13.0__py3-none-any.whl → 13.1.0__py3-none-any.whl

ml_tools/_schema.py

@@ -0,0 +1,19 @@
+from typing import NamedTuple, Tuple, Optional, Dict
+
+class FeatureSchema(NamedTuple):
+    """Holds the final, definitive schema for the model pipeline."""
+    
+    # The final, ordered list of all feature names
+    feature_names: Tuple[str, ...]
+    
+    # List of all continuous feature names
+    continuous_feature_names: Tuple[str, ...]
+    
+    # List of all categorical feature names
+    categorical_feature_names: Tuple[str, ...]
+    
+    # Map of {column_index: cardinality} for categorical features
+    categorical_index_map: Optional[Dict[int, int]]
+    
+    # The original string-to-int mappings (e.g., {'color': {'red': 0, 'blue': 1}})
+    categorical_mappings: Optional[Dict[str, Dict[str, int]]]

ml_tools/data_exploration.py

@@ -11,7 +11,7 @@ from .path_manager import sanitize_filename, make_fullpath
 from ._script_info import _script_info
 from ._logger import _LOGGER
 from .utilities import save_dataframe_filename
-
+from ._schema import FeatureSchema
 
 # Keep track of all available tools, show using `info()`
 __all__ = [
@@ -32,9 +32,9 @@ __all__ = [
     "drop_outlier_samples",
     "match_and_filter_columns_by_regex",
     "standardize_percentages",
-    "create_transformer_categorical_map",
     "reconstruct_one_hot",
-    "reconstruct_binary"
+    "reconstruct_binary",
+    "finalize_feature_schema"
 ]
 
 
@@ -977,49 +977,6 @@ def standardize_percentages(
     return df_copy
 
 
-def create_transformer_categorical_map(
-    df: pd.DataFrame,
-    mappings: Dict[str, Dict[str, int]],
-    verbose: bool = True
-) -> Dict[int, int]:
-    """
-    Creates the `categorical_map` required by a `TabularTransformer` model.
-
-    This function should be called late in the preprocessing pipeline, after all
-    column additions, deletions, or reordering have occurred. It uses the final
-    DataFrame's column order to map the correct column index to its cardinality.
-
-    Args:
-        df (pd.DataFrame): The final, processed DataFrame.
-        mappings (Dict[str, Dict[str, int]]): The mappings dictionary generated by
-          `encode_categorical_features`, containing the category-to-integer
-          mapping for each categorical column.
-        verbose (bool): If True, prints mapping progress.
-
-    Returns:
-        (Dict[int, int]): The final `categorical_map` for the transformer,
-          mapping each column's current index to its cardinality (e.g., {0: 3}).
-    """
-    transformer_map = {}
-    categorical_column_names = mappings.keys()
-
-    _LOGGER.info("Creating categorical map for TabularTransformer.")
-    for col_name in categorical_column_names:
-        if col_name in df.columns:
-            col_idx = df.columns.get_loc(col_name)
-            
-            # Get cardinality directly from the length of the mapping dictionary
-            cardinality = len(mappings[col_name])
-            
-            transformer_map[col_idx] = cardinality
-            if verbose:
-                print(f"  - Mapping column '{col_name}' at index {col_idx} with cardinality {cardinality}.")
-        else:
-            _LOGGER.warning(f"Categorical column '{col_name}' not found in the final DataFrame. Skipping.")
-            
-    return transformer_map
-
-
 def reconstruct_one_hot(
     df: pd.DataFrame,
     features_to_reconstruct: List[Union[str, Tuple[str, Optional[str]]]],
@@ -1274,6 +1231,78 @@ def reconstruct_binary(
     return new_df
 
 
+def finalize_feature_schema(
+    df_features: pd.DataFrame,
+    categorical_mappings: Optional[Dict[str, Dict[str, int]]]
+) -> FeatureSchema:
+    """
+    Analyzes the final features DataFrame to create a definitive schema.
+
+    This function is the "single source of truth" for column order
+    and type (categorical vs. continuous) for the entire ML pipeline.
+
+    It should be called at the end of the feature engineering process.
+
+    Args:
+        df_features (pd.DataFrame):
+            The final, processed DataFrame containing *only* feature columns
+            in the exact order they will be fed to the model.
+        categorical_mappings (Dict[str, Dict[str, int]] | None):
+            The mappings dictionary generated by
+            `encode_categorical_features`. Can be None if no
+            categorical features exist.
+
+    Returns:
+        FeatureSchema: A NamedTuple containing all necessary metadata for the pipeline.
+    """
+    feature_names: List[str] = df_features.columns.to_list()
+    
+    # Intermediate lists for building
+    continuous_feature_names_list: List[str] = []
+    categorical_feature_names_list: List[str] = []
+    categorical_index_map_dict: Dict[int, int] = {}
+
+    _LOGGER.info("Finalizing feature schema...")
+
+    if categorical_mappings:
+        # --- Categorical features are present ---
+        categorical_names_set = set(categorical_mappings.keys())
+        
+        for index, name in enumerate(feature_names):
+            if name in categorical_names_set:
+                # This is a categorical feature
+                cardinality = len(categorical_mappings[name])
+                categorical_index_map_dict[index] = cardinality
+                categorical_feature_names_list.append(name)
+            else:
+                # This is a continuous feature
+                continuous_feature_names_list.append(name)
+        
+        # Use the populated dict, or None if it's empty
+        final_index_map = categorical_index_map_dict if categorical_index_map_dict else None
+    
+    else:
+        # --- No categorical features ---
+        _LOGGER.info("No categorical mappings provided. Treating all features as continuous.")
+        continuous_feature_names_list = list(feature_names)
+        # categorical_feature_names_list remains empty
+        # categorical_index_map_dict remains empty
+        final_index_map = None # Explicitly set to None to match Optional type
+
+    _LOGGER.info(f"Schema created: {len(continuous_feature_names_list)} continuous, {len(categorical_feature_names_list)} categorical.")
+    
+    # Create the final immutable instance
+    schema_instance = FeatureSchema(
+        feature_names=tuple(feature_names),
+        continuous_feature_names=tuple(continuous_feature_names_list),
+        categorical_feature_names=tuple(categorical_feature_names_list),
+        categorical_index_map=final_index_map,
+        categorical_mappings=categorical_mappings
+    )
+    
+    return schema_instance
+
+
 def _validate_columns(df: pd.DataFrame, columns: list[str]):
     valid_columns = [column for column in columns if column in df.columns]
     return valid_columns

ml_tools/keys.py

@@ -68,6 +68,15 @@ class SHAPKeys:
     SAVENAME = "shap_summary"
 
 
+class PyTorchCheckpointKeys:
+    """Keys for saving/loading a training checkpoint dictionary."""
+    MODEL_STATE = "model_state_dict"
+    OPTIMIZER_STATE = "optimizer_state_dict"
+    SCHEDULER_STATE = "scheduler_state_dict"
+    EPOCH = "epoch"
+    BEST_SCORE = "best_score"
+
+
 class _OneHotOtherPlaceholder:
     """Used internally by GUI_tools."""
     OTHER_GUI = "OTHER"

ml_tools/optimization_tools.py

@@ -9,6 +9,7 @@ from .utilities import yield_dataframes_from_dir
 from ._logger import _LOGGER
 from ._script_info import _script_info
 from .SQL import DatabaseManager
+from ._schema import FeatureSchema
 
 
 __all__ = [
@@ -19,35 +20,25 @@ __all__ = [
 
 
 def create_optimization_bounds(
-    csv_path: Union[str, Path],
+    schema: FeatureSchema,
     continuous_bounds_map: Dict[str, Tuple[float, float]],
-    categorical_map: Dict[int, int],
-    target_column: Optional[str] = None,
     start_at_zero: bool = True
 ) -> Tuple[List[float], List[float]]:
     """
-    Generates the lower and upper bounds lists for the optimizer from a CSV header.
+    Generates the lower and upper bounds lists for the optimizer from a FeatureSchema.
 
     This helper function automates the creation of unbiased bounds for
     categorical features and combines them with user-defined bounds for
-    continuous features.
-
-    It reads *only* the header of the provided CSV to determine the full
-    list of feature columns and their order, excluding the specified target.
-    This is memory-efficient as the full dataset is not loaded.
+    continuous features, using the schema as the single source of truth
+    for feature order and type.
 
     Args:
-        csv_path (Union[str, Path]):
-            Path to the final, preprocessed CSV file. The column order in
-            this file must match the order expected by the model.
+        schema (FeatureSchema):
+            The definitive schema object created by 
+            `data_exploration.finalize_feature_schema()`.
         continuous_bounds_map (Dict[str, Tuple[float, float]]):
             A dictionary mapping the *name* of each **continuous** feature
             to its (min_bound, max_bound) tuple.
-        categorical_map (Dict[int, int]):
-            The map from the *index* of each **categorical** feature to its cardinality.
-            (e.g., {2: 4} for a feature at index 2 with 4 categories).
-        target_column (Optional[str], optional):
-            The name of the target column to exclude. If None (default), the *last column* in the CSV is assumed to be the target.
         start_at_zero (bool):
             - If True, assumes categorical encoding is [0, 1, ..., k-1].
               Bounds will be set as [-0.5, k - 0.5].
@@ -59,98 +50,86 @@ def create_optimization_bounds(
             A tuple containing two lists: (lower_bounds, upper_bounds).
 
     Raises:
-        ValueError: If a feature is defined in both maps, is missing from
-                    both maps, or if a name in `continuous_bounds_map`
-                    or `target_column` is not found in the CSV columns.
+        ValueError: If a feature is missing from `continuous_bounds_map`
+                    or if a feature name in the map is not a
+                    continuous feature according to the schema.
     """
-    # 1. Read header and determine feature names
-    full_csv_path = make_fullpath(csv_path, enforce="file")
-    try:
-        df_header = pd.read_csv(full_csv_path, nrows=0, encoding="utf-8")
-    except Exception as e:
-        _LOGGER.error(f"Failed to read header from CSV: {e}")
-        raise
-        
-    all_column_names = df_header.columns.to_list()
-    feature_names: List[str] = []
-
-    if target_column is None:
-        feature_names = all_column_names[:-1]
-        excluded_target = all_column_names[-1]
-        _LOGGER.info(f"No target_column provided. Assuming last column '{excluded_target}' is the target.")
-    else:
-        if target_column not in all_column_names:
-            _LOGGER.error(f"Target column '{target_column}' not found in CSV header.")
-            raise ValueError()
-        feature_names = [name for name in all_column_names if name != target_column]
-        _LOGGER.info(f"Excluding target column '{target_column}'.")
-    
-    # 2. Initialize bound lists
+    # 1. Get feature names and map from schema
+    feature_names = schema.feature_names
+    categorical_index_map = schema.categorical_index_map
     total_features = len(feature_names)
+
     if total_features <= 0:
-        _LOGGER.error("No feature columns remain after excluding the target.")
+        _LOGGER.error("Schema contains no features.")
         raise ValueError()
+        
+    _LOGGER.info(f"Generating bounds for {total_features} total features...")
 
+    # 2. Initialize bound lists
     lower_bounds: List[Optional[float]] = [None] * total_features
     upper_bounds: List[Optional[float]] = [None] * total_features
-    
-    _LOGGER.info(f"Generating bounds for {total_features} total features...")
 
     # 3. Populate categorical bounds (Index-based)
-    # The indices in categorical_map (e.g., {2: 4}) directly correspond
-    # to the indices in the `feature_names` list.
-    for index, cardinality in categorical_map.items():
-        if not (0 <= index < total_features):
-            _LOGGER.error(f"Categorical index {index} is out of range for the {total_features} features.")
-            raise ValueError()
-            
-        if start_at_zero:
-            # Rule for [0, k-1]: bounds are [-0.5, k - 0.5]
-            low = -0.5
-            high = float(cardinality) - 0.5
-        else:
-            # Rule for [1, k]: bounds are [0.5, k + 0.5]
-            low = 0.5
-            high = float(cardinality) + 0.5
-            
-        lower_bounds[index] = low
-        upper_bounds[index] = high
+    if categorical_index_map:
+        for index, cardinality in categorical_index_map.items():
+            if not (0 <= index < total_features):
+                _LOGGER.error(f"Categorical index {index} is out of range for the {total_features} features.")
+                raise ValueError()
+                
+            if start_at_zero:
+                # Rule for [0, k-1]: bounds are [-0.5, k - 0.5]
+                low = -0.5
+                high = float(cardinality) - 0.5
+            else:
+                # Rule for [1, k]: bounds are [0.5, k + 0.5]
+                low = 0.5
+                high = float(cardinality) + 0.5
+                
+            lower_bounds[index] = low
+            upper_bounds[index] = high
         
-    _LOGGER.info(f"Automatically set bounds for {len(categorical_map)} categorical features.")
+        _LOGGER.info(f"Automatically set bounds for {len(categorical_index_map)} categorical features.")
+    else:
+        _LOGGER.info("No categorical features found in schema.")
 
     # 4. Populate continuous bounds (Name-based)
+    # Use schema.continuous_feature_names for robust checking
+    continuous_names_set = set(schema.continuous_feature_names)
+    
+    if continuous_names_set != set(continuous_bounds_map.keys()):
+        missing_in_map = continuous_names_set - set(continuous_bounds_map.keys())
+        if missing_in_map:
+            _LOGGER.error(f"The following continuous features are missing from 'continuous_bounds_map': {list(missing_in_map)}")
+        
+        extra_in_map = set(continuous_bounds_map.keys()) - continuous_names_set
+        if extra_in_map:
+            _LOGGER.error(f"The following features in 'continuous_bounds_map' are not defined as continuous in the schema: {list(extra_in_map)}")
+            
+        raise ValueError("Mismatch between 'continuous_bounds_map' and schema's continuous features.")
+
     count_continuous = 0
     for name, (low, high) in continuous_bounds_map.items():
-        try:
-            # Map name to its index in the *feature-only* list
-            index = feature_names.index(name)
-        except ValueError:
-            _LOGGER.warning(f"Feature name '{name}' from 'continuous_bounds_map' not found in the CSV's feature columns.")
-            continue
-            
+        # Map name to its index in the *feature-only* list
+        # This is guaranteed to be correct by the schema
+        index = feature_names.index(name)
+
         if lower_bounds[index] is not None:
-            # This index was already set by the categorical map
-            _LOGGER.error(f"Feature '{name}' (at index {index}) is defined in both 'categorical_map' and 'continuous_bounds_map'.")
+            # This should be impossible if schema is correct, but good to check
+            _LOGGER.error(f"Schema conflict: Feature '{name}' (at index {index}) is defined as both continuous and categorical.")
             raise ValueError()
-            
+
         lower_bounds[index] = float(low)
         upper_bounds[index] = float(high)
         count_continuous += 1
         
     _LOGGER.info(f"Manually set bounds for {count_continuous} continuous features.")
 
-    # 5. Validation: Check for any remaining None values
-    missing_indices = []
-    for i in range(total_features):
-        if lower_bounds[i] is None:
-            missing_indices.append(i)
-            
-    if missing_indices:
+    # 5. Final Validation (all Nones should be filled)
+    if None in lower_bounds:
+        missing_indices = [i for i, b in enumerate(lower_bounds) if b is None]
         missing_names = [feature_names[i] for i in missing_indices]
-        _LOGGER.error(f"Bounds not defined for all features. Missing: {missing_names}")
-        raise ValueError()
-
-    # _LOGGER.info("All bounds successfully created.")
+        _LOGGER.error(f"Failed to create all bounds. This indicates an internal logic error. Missing: {missing_names}")
+        raise RuntimeError("Internal error: Not all bounds were populated.")
     
     # Cast to float lists, as 'None' sentinels are gone
     return (

ml_tools/serde.py

@@ -116,8 +116,7 @@ def deserialize_object(
             # Can't do an isinstance check on 'Any', skip it.
             if type_to_check is not Any and not isinstance(obj, type_to_check):
                 error_msg = (
-                    f"Type mismatch: Expected an instance of '{expected_type}', "
-                    f"but found '{type(obj)}' in '{true_filepath}'."
+                    f"Type mismatch: Expected an instance of '{expected_type}', but found '{type(obj)}' in '{true_filepath}'."
                 )
                 _LOGGER.error(error_msg)
                 raise TypeError()

dragon_ml_toolbox-12.13.0.dist-info/RECORD

@@ -1,41 +0,0 @@
-dragon_ml_toolbox-12.13.0.dist-info/licenses/LICENSE,sha256=L35WDmmLZNTlJvxF6Vy7Uy4SYNi6rCfWUqlTHpoRMoU,1081
-dragon_ml_toolbox-12.13.0.dist-info/licenses/LICENSE-THIRD-PARTY.md,sha256=iy2r_R7wjzsCbz_Q_jMsp_jfZ6oP8XW9QhwzRBH0mGY,1904
-ml_tools/ETL_cleaning.py,sha256=2VBRllV8F-ZiPylPp8Az2gwn5ztgazN0BH5OKnRUhV0,20402
-ml_tools/ETL_engineering.py,sha256=KfYqgsxupAx6e_TxwO1LZXeu5mFkIhVXJrNjP3CzIZc,54927
-ml_tools/GUI_tools.py,sha256=Va6ig-dHULPVRwQYYtH3fvY5XPIoqRcJpRW8oXC55Hw,45413
-ml_tools/MICE_imputation.py,sha256=X273Qlgoqqg7KTmoKd75YDyAPB0UIbTzGP3xsCmRh3E,11717
-ml_tools/ML_callbacks.py,sha256=2ZazJjlbClP-ALc8q0ru2oalkugbhO3TFwPg4RFZpck,14056
-ml_tools/ML_datasetmaster.py,sha256=kedCGneR3S2zui0_JFZN6TBL5e69XWkdpkE_QohyqSM,31433
-ml_tools/ML_evaluation.py,sha256=h7fAtk0lS4gTqQ46fiVjucTvFlX4rsufKnEtate6Nu0,18381
-ml_tools/ML_evaluation_multi.py,sha256=Kn9n5lfxo7A0TvgIDMx8UHZCvzTqv1ViezzwJBF-ypM,15970
-ml_tools/ML_inference.py,sha256=ymFvncFsU10PExq87xnEj541DKV5ck0nMuK8ToJHzVQ,23067
-ml_tools/ML_models.py,sha256=G64NPhYZfYvHTIUwkIrMrNLgfDTKJwqdc8jwesPqB9E,28090
-ml_tools/ML_optimization.py,sha256=es3TlQbY7RYgJMZnznkjYGbUxFnAqzZxE_g3_qLK9Q8,22960
-ml_tools/ML_scaler.py,sha256=tw6onj9o8_kk3FQYb930HUzvv1zsFZe2YZJdF3LtHkU,7538
-ml_tools/ML_simple_optimization.py,sha256=W2mce1XFCuiOHTOjOsCNbETISHn5MwYlYsTIXH5hMMo,18177
-ml_tools/ML_trainer.py,sha256=UmCuKr_GzQGYqhEZ-kaRv9Buj44DsNyuOzmOM7Fw8N0,24569
-ml_tools/ML_utilities.py,sha256=EnKpPTnJ2qjZmz7kvows4Uu5CfSA7ByRmI1v2-KarKw,9337
-ml_tools/PSO_optimization.py,sha256=fVHeemqilBS0zrGV25E5yKwDlGdd2ZKa18d8CZ6Q6Fk,22961
-ml_tools/RNN_forecast.py,sha256=Qa2KoZfdAvSjZ4yE78N4BFXtr3tTr0Gx7tQJZPotsh0,1967
-ml_tools/SQL.py,sha256=vXLPGfVVg8bfkbBE3HVfyEclVbdJy0TBhuQONtMwSCQ,11234
-ml_tools/VIF_factor.py,sha256=at5IVqPvicja2-DNSTSIIy3SkzDWCmLzo3qTG_qr5n8,10422
-ml_tools/__init__.py,sha256=q0y9faQ6e17XCQ7eUiCZ1FJ4Bg5EQqLjZ9f_l5REUUY,41
-ml_tools/_logger.py,sha256=dlp5cGbzooK9YSNSZYB4yjZrOaQUGW8PTrM411AOvL8,4717
-ml_tools/_script_info.py,sha256=21r83LV3RubsNZ_RTEUON6RbDf7Mh4_udweNcvdF_Fk,212
-ml_tools/constants.py,sha256=3br5Rk9cL2IUo638eJuMOGdbGQaWssaUecYEvSeRBLM,3322
-ml_tools/custom_logger.py,sha256=7tSAgRL7e-Ekm7rS1FLDocaPLCnaoKc7VSrtfwCtCEg,10067
-ml_tools/data_exploration.py,sha256=haddQFsXAWzuf84NLItcZ4Q7vzN3YWjFoh7lPlWUczo,50679
-ml_tools/ensemble_evaluation.py,sha256=FGHSe8LBI8_w8LjNeJWOcYQ1UK_mc6fVah8gmSvNVGg,26853
-ml_tools/ensemble_inference.py,sha256=0yLmLNj45RVVoSCLH1ZYJG9IoAhTkWUqEZmLOQTFGTY,9348
-ml_tools/ensemble_learning.py,sha256=vsIED7nlheYI4w2SBzP6SC1AnNeMfn-2A1Gqw5EfxsM,21964
-ml_tools/handle_excel.py,sha256=pfdAPb9ywegFkM9T54bRssDOsX-K7rSeV0RaMz7lEAo,14006
-ml_tools/keys.py,sha256=FDpbS3Jb0pjrVvvp2_8nZi919mbob_-xwuy5OOtKM_A,1848
-ml_tools/math_utilities.py,sha256=PxoOrnuj6Ntp7_TJqyDWi0JX03WpAO5iaFNK2Oeq5I4,8800
-ml_tools/optimization_tools.py,sha256=P074YCuZzkqkONnAsM-Zb9DTX_i8cRkkJLpwAWz6CRw,13521
-ml_tools/path_manager.py,sha256=CyDU16pOKmC82jPubqJPT6EBt-u-3rGVbxyPIZCvDDY,18432
-ml_tools/serde.py,sha256=ll2mVC0sO2jIEdG3K6xMcgEN13N4YSb8VjviGvw_ers,4949
-ml_tools/utilities.py,sha256=OcAyV1tEcYAfOWlGjRgopsjDLxU3DcI5EynzvWV4q3A,15754
-dragon_ml_toolbox-12.13.0.dist-info/METADATA,sha256=p3-oOSqq1hhJj13KjIXeFnwBu3UTfBJu5mTDL9MCpdU,6167
-dragon_ml_toolbox-12.13.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dragon_ml_toolbox-12.13.0.dist-info/top_level.txt,sha256=wm-oxax3ciyez6VoO4zsFd-gSok2VipYXnbg3TH9PtU,9
-dragon_ml_toolbox-12.13.0.dist-info/RECORD,,