dragon-ml-toolbox 20.0.0__tar.gz → 20.1.0__tar.gz

{dragon_ml_toolbox-20.0.0/dragon_ml_toolbox.egg-info → dragon_ml_toolbox-20.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 20.0.0
+Version: 20.1.0
 Summary: Complete pipelines and helper tools for data science and machine learning projects.
 Author-email: Karl Luigi Loza Vidaurre <luigiloza@gmail.com>
 License-Expression: MIT

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0/dragon_ml_toolbox.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dragon-ml-toolbox
-Version: 20.0.0
+Version: 20.1.0
 Summary: Complete pipelines and helper tools for data science and machine learning projects.
 Author-email: Karl Luigi Loza Vidaurre <luigiloza@gmail.com>
 License-Expression: MIT

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/dragon_ml_toolbox.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ ml_tools/ML_chain/__init__.py
 ml_tools/ML_chain/_chaining_tools.py
 ml_tools/ML_chain/_dragon_chain.py
 ml_tools/ML_chain/_imprimir.py
+ml_tools/ML_chain/_update_schema.py
 ml_tools/ML_configuration/__init__.py
 ml_tools/ML_configuration/_base_model_config.py
 ml_tools/ML_configuration/_finalize.py

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/MICE/_dragon_mice.py

@@ -197,7 +197,7 @@ class DragonMICE:
                  _LOGGER.error(f"Index mismatch in dataset {subname}")
                  raise ValueError()
         
-        _LOGGER.info("Schema-based MICE imputation complete.")
+        _LOGGER.info("⬅️ Schema-based MICE imputation complete.")
         
         return kernel, imputed_datasets, imputed_dataset_names
         
@@ -237,9 +237,6 @@ class DragonMICE:
                 # We pass an empty DF as 'targets' to save_imputed_datasets to prevent duplication.
                 df_input = df
                 df_targets_to_save = pd.DataFrame(index=df.index) 
-                
-                # Monitor all columns that had NaNs
-                imputed_column_names = [col for col in df.columns if df[col].isna().any()]
             else:
                 # Explicitly cast tuple to list for Pandas indexing
                 feature_cols = list(self._schema.feature_names)
@@ -253,8 +250,9 @@ class DragonMICE:
                 df_input = df[feature_cols]
                 # Drop features to get targets (more robust than explicit selection if targets vary)
                 df_targets_to_save = df.drop(columns=feature_cols)
-                
-                imputed_column_names = _get_na_column_names(df=df_input) # type: ignore
+            
+            # Monitor all columns that had NaNs
+            imputed_column_names = [col for col in df_input.columns if df_input[col].isna().any()]
 
             # Run core logic
             kernel, imputed_datasets, imputed_dataset_names = self._run_mice(df=df_input, df_name=df_name) # type: ignore
@@ -316,35 +314,41 @@ def get_convergence_diagnostic(kernel: mf.ImputationKernel, imputed_dataset_name
     
     # iterate over each imputed dataset
     for dataset_id, imputed_dataset_name in zip(range(dataset_count), imputed_dataset_names):
-        #Check directory for current dataset
         dataset_file_dir = f"Convergence_Metrics_{imputed_dataset_name}"
         local_save_dir = make_fullpath(input_path=root_path / dataset_file_dir, make=True)
         
-        for feature_name in column_names:
-            means_per_iteration = []
-            for iteration in range(iterations_cap):
-                current_imputed = kernel.complete_data(dataset=dataset_id, iteration=iteration)
-                means_per_iteration.append(np.mean(current_imputed[feature_name])) # type: ignore
-                
+        # 1. Pre-calculate means for all features across all iterations
+        # Structure: {feature_name: [mean_iter_0, mean_iter_1, ...]}
+        history = {col: [] for col in column_names}
+        
+        for iteration in range(iterations_cap):
+            # Resolve dataset ONLY ONCE per iteration
+            current_imputed = kernel.complete_data(dataset=dataset_id, iteration=iteration)
+            
+            for col in column_names:
+                # Fast lookup
+                val = np.mean(current_imputed[col])
+                history[col].append(val)
+
+        # 2. Plotting loop
+        for feature_name, means_per_iteration in history.items():
             plt.figure(figsize=(10, 8))
             plt.plot(means_per_iteration, marker='o')
             plt.xlabel("Iteration", **label_font)
             plt.ylabel("Mean of Imputed Values", **label_font)
             plt.title(f"Mean Convergence for '{feature_name}'", **label_font)
             
-            # Adjust plot display for the X axis
             _ticks = np.arange(iterations_cap)
             _labels = np.arange(1, iterations_cap + 1)
-            plt.xticks(ticks=_ticks, labels=_labels) # type: ignore
+            plt.xticks(ticks=_ticks, labels=_labels)
             plt.grid(True)
             
-            feature_save_name = sanitize_filename(feature_name)
-            feature_save_name = feature_save_name + ".svg"
+            feature_save_name = sanitize_filename(feature_name) + ".svg"
             save_path = local_save_dir / feature_save_name
             plt.savefig(save_path, bbox_inches='tight', format="svg")
             plt.close()
             
-        _LOGGER.info(f"{dataset_file_dir} process completed.")
+    _LOGGER.info(f"📉 Convergence diagnostics complete.")
 
 
 # Imputed distributions
@@ -431,5 +435,5 @@ def get_imputed_distributions(kernel: mf.ImputationKernel, df_name: str, root_di
             fig = kernel.plot_imputed_distributions(variables=[feature])
             _process_figure(fig, feature)
 
-    _LOGGER.info(f"{local_dir_name} completed.")
+    _LOGGER.info(f"📊 Imputed distributions complete.")
     

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/ML_chain/__init__.py

@@ -8,11 +8,16 @@ from ._chaining_tools import (
     prepare_chaining_dataset,
 )
 
+from ._update_schema import (
+    derive_next_step_schema
+)
+
 from ._imprimir import info
 
 
 __all__ = [
     "DragonChainOrchestrator",
+    "derive_next_step_schema",
     "augment_dataset_with_predictions",
     "augment_dataset_with_predictions_multi",
     "prepare_chaining_dataset",

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/ML_chain/_chaining_tools.py

@@ -5,7 +5,7 @@ from typing import Optional, Literal
 
 from ..ML_inference import DragonInferenceHandler
 
-from ..keys._keys import MLTaskKeys, PyTorchInferenceKeys
+from ..keys._keys import MLTaskKeys, PyTorchInferenceKeys, ChainKeys
 from .._core import get_logger
 
 
@@ -23,11 +23,10 @@ def augment_dataset_with_predictions(
     handler: DragonInferenceHandler,
     dataset: pd.DataFrame,
     ground_truth_targets: list[str],
-    prediction_col_prefix: str = "pred_",
     batch_size: int = 4096
 ) -> pd.DataFrame:
     """
-    Uses a DragonInferenceHandler to generate predictions for a dataset and appends them as new feature columns.
+    Uses a DragonInferenceHandler to generate predictions for a dataset and appends them as new feature columns with a standardized prefix.
     
     This function splits the features from the ground truth targets, runs inference in batches to ensure
     memory efficiency, and returns a unified DataFrame containing:
@@ -38,8 +37,6 @@ def augment_dataset_with_predictions(
         dataset (pd.DataFrame): The input pandas DataFrame containing features and ground truth targets.
         ground_truth_targets (List[str]): A list of column names in `dataset` representing the actual targets.
             These are removed from the input features during inference and appended to the end of the result.
-        prediction_col_prefix (str, optional): A string to prepend when creating the
-            new prediction columns.
         batch_size (int, optional): The number of samples to process in a single inference step. 
             Prevents OOM errors on large datasets. Defaults to 4096.
 
@@ -107,7 +104,7 @@ def augment_dataset_with_predictions(
     full_prediction_array = np.vstack(all_predictions)
     
     # Generate new column names
-    new_col_names = [f"{prediction_col_prefix}{tid}" for tid in handler.target_ids]
+    new_col_names = [f"{ChainKeys.CHAIN_PREDICTION_PREFIX}{tid}" for tid in handler.target_ids]
     
     # Verify dimensions match
     if full_prediction_array.shape[1] != len(new_col_names):

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/ML_chain/_dragon_chain.py

@@ -77,18 +77,16 @@ class DragonChainOrchestrator:
     def update_with_inference(
         self, 
         handler: DragonInferenceHandler, 
-        prefix: str = "pred_", 
         batch_size: int = 4096
     ) -> None:
         """
         Runs inference using the provided handler on the full internal dataset and appends the results as new features.
         
         This updates the internal state of the Orchestrator. Subsequent calls to `get_training_data` 
-        will include these new prediction columns as features.
+        will include these new prediction columns as features with a standardized prefix.
 
         Args:
             handler (DragonInferenceHandler): The trained model handler.
-            prefix (str): Prefix for the new prediction columns (e.g., "m1_", "step2_").
             batch_size (int): Batch size for inference.
         """
         _LOGGER.info(f"Orchestrator: Updating internal state with predictions from handler (Targets: {handler.target_ids})...")
@@ -99,7 +97,6 @@ class DragonChainOrchestrator:
             handler=handler,
             dataset=self.current_dataset,
             ground_truth_targets=self.all_targets,
-            prediction_col_prefix=prefix,
             batch_size=batch_size
         )
         

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/ML_chain/_imprimir.py

@@ -2,6 +2,7 @@ from .._core import _imprimir_disponibles
 
 _GRUPOS = [
     "DragonChainOrchestrator",
+    "derive_next_step_schema",
     "augment_dataset_with_predictions",
     "augment_dataset_with_predictions_multi",
     "prepare_chaining_dataset",

dragon_ml_toolbox-20.1.0/ml_tools/ML_chain/_update_schema.py

@@ -0,0 +1,96 @@
+from ..schema import FeatureSchema
+from ..ML_inference import DragonInferenceHandler
+
+from ..keys._keys import MLTaskKeys, ChainKeys
+from .._core import get_logger
+
+
+_LOGGER = get_logger("Schema Updater")
+
+
+__all__ = [
+    "derive_next_step_schema",
+]
+
+
+def derive_next_step_schema(
+    current_schema: FeatureSchema, 
+    handler: DragonInferenceHandler, 
+    verbose: bool = True
+) -> FeatureSchema:
+    """
+    Creates the FeatureSchema for the NEXT step in the chain by appending the current handler's predictions as new features.
+    
+    Args:
+        current_schema (FeatureSchema): The current FeatureSchema.
+        handler (DragonInferenceHandler): The inference handler of the model trained using the current schema.
+    
+    Returns:
+        FeatureSchema: An updated schema including new predicted features.
+    """
+    # 1. Determine New Column Names
+    # Match logic from _chaining_tools.py
+    if handler.target_ids is None:
+        _LOGGER.error("Handler target_ids is None; cannot derive schema.")
+        raise ValueError()
+        
+    new_cols = [f"{ChainKeys.CHAIN_PREDICTION_PREFIX}{tid}" for tid in handler.target_ids]
+    
+    # 2. Base Lists (Convert tuples to lists for mutation)
+    new_feature_names = list(current_schema.feature_names) + new_cols
+    new_cont_names = list(current_schema.continuous_feature_names)
+    new_cat_names = list(current_schema.categorical_feature_names)
+    
+    # Copy existing maps (handle None case)
+    new_cat_index_map = dict(current_schema.categorical_index_map) if current_schema.categorical_index_map else {}
+    new_cat_mappings = dict(current_schema.categorical_mappings) if current_schema.categorical_mappings else {}
+
+    # 3. Determine Feature Type based on Task
+    is_categorical = False
+    cardinality = 0
+    
+    if handler.task in [MLTaskKeys.BINARY_CLASSIFICATION, MLTaskKeys.MULTILABEL_BINARY_CLASSIFICATION]:
+        is_categorical = True
+        cardinality = 2
+        
+    elif handler.task == MLTaskKeys.MULTICLASS_CLASSIFICATION:
+        is_categorical = True
+        # We rely on the class map to know the 'vocabulary' size
+        if handler._class_map is None:
+            _LOGGER.error("Handler class_map is None, cannot determine cardinality for multiclass classification model.")
+            raise ValueError()
+        cardinality = len(handler._class_map)
+        
+    # 4. Append New Metadata
+    current_total_feats = len(current_schema.feature_names)
+    
+    for i, col_name in enumerate(new_cols):
+        # Calculate the absolute index of this new column
+        # If we had 10 features (0-9), the new one is at index 10 + i
+        new_index = current_total_feats + i
+        
+        if is_categorical:
+            new_cat_names.append(col_name)
+            
+            # A. Update Cardinality for Embeddings
+            new_cat_index_map[new_index] = cardinality
+            
+            # B. Create Identity Mapping (Dummy Encoding)
+            # Maps string representation of int back to the int. 
+            identity_map = {str(k): k for k in range(cardinality)}
+            new_cat_mappings[col_name] = identity_map
+        else:
+            # Regression / Multitarget Regression
+            new_cont_names.append(col_name)
+    
+    if verbose:
+        _LOGGER.info(f"Derived next step schema with {len(new_feature_names)} features:\n    {len(new_cont_names)} continuous\n    {len(new_cat_names)} categorical")
+
+    # 5. Return New Immutable Schema
+    return FeatureSchema(
+        feature_names=tuple(new_feature_names),
+        continuous_feature_names=tuple(new_cont_names),
+        categorical_feature_names=tuple(new_cat_names),
+        categorical_index_map=new_cat_index_map if new_cat_index_map else None,
+        categorical_mappings=new_cat_mappings if new_cat_mappings else None
+    )

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/keys/_keys.py

@@ -278,6 +278,11 @@ class SchemaKeys:
     OPTIONAL_LABELS = "optional_labels"
 
 
+class ChainKeys:
+    """Used by the ML chaining module."""
+    CHAIN_PREDICTION_PREFIX = "pred_"
+
+
 class _EvaluationConfig:
     """Set config values for evaluation modules."""
     DPI = 400

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/schema/_feature_schema.py

@@ -44,7 +44,7 @@ class FeatureSchema(NamedTuple):
         Handles conversion of Tuple->List and IntKeys->StrKeys automatically.
         """
         # validate path
-        dir_path = make_fullpath(directory, enforce="directory")
+        dir_path = make_fullpath(directory, make=True, enforce="directory")
         file_path = dir_path / SchemaKeys.SCHEMA_FILENAME
         
         try:

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dragon-ml-toolbox"
-version = "20.0.0"
+version = "20.1.0"
 description = "Complete pipelines and helper tools for data science and machine learning projects."
 authors = [
     { name = "Karl Luigi Loza Vidaurre", email = "luigiloza@gmail.com" }

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/data_exploration/__init__.py

@@ -53,13 +53,13 @@ __all__ = [
     "split_features_targets", 
     "split_continuous_binary", 
     "split_continuous_categorical_targets",
-    "encode_categorical_features",
     "clip_outliers_single", 
     "clip_outliers_multi",
     "drop_outlier_samples",
     "plot_continuous_vs_target",
     "plot_categorical_vs_target",
     "plot_correlation_heatmap", 
+    "encode_categorical_features",
     "finalize_feature_schema",
     "apply_feature_schema",
     "match_and_filter_columns_by_regex",

{dragon_ml_toolbox-20.0.0 → dragon_ml_toolbox-20.1.0}/ml_tools/data_exploration/_imprimir.py

@@ -12,13 +12,13 @@ _GRUPOS = [
     "split_features_targets", 
     "split_continuous_binary", 
     "split_continuous_categorical_targets",
-    "encode_categorical_features",
     "clip_outliers_single", 
     "clip_outliers_multi",
     "drop_outlier_samples",
     "plot_continuous_vs_target",
     "plot_categorical_vs_target",
     "plot_correlation_heatmap", 
+    "encode_categorical_features",
     "finalize_feature_schema",
     "apply_feature_schema",
     "match_and_filter_columns_by_regex",