dragon-ml-toolbox 10.1.1__py3-none-any.whl → 14.2.0__py3-none-any.whl

ml_tools/optimization_tools.py

@@ -1,6 +1,6 @@
 import matplotlib.pyplot as plt
 import seaborn as sns
-from typing import Union, Any, Literal, Optional
+from typing import Union, Any, Literal, Optional, Dict, List, Tuple
 from pathlib import Path
 import pandas as pd
 
@@ -9,14 +9,135 @@ from .utilities import yield_dataframes_from_dir
 from ._logger import _LOGGER
 from ._script_info import _script_info
 from .SQL import DatabaseManager
+from ._schema import FeatureSchema
 
 
 __all__ = [
+    "create_optimization_bounds",
     "parse_lower_upper_bounds",
-    "plot_optimal_feature_distributions"
+    "plot_optimal_feature_distributions",
 ]
 
 
+def create_optimization_bounds(
+    schema: FeatureSchema,
+    continuous_bounds_map: Dict[str, Tuple[float, float]],
+    start_at_zero: bool = True
+) -> Tuple[List[float], List[float]]:
+    """
+    Generates the lower and upper bounds lists for the optimizer from a FeatureSchema.
+
+    This helper function automates the creation of unbiased bounds for
+    categorical features and combines them with user-defined bounds for
+    continuous features, using the schema as the single source of truth
+    for feature order and type.
+
+    Args:
+        schema (FeatureSchema):
+            The definitive schema object created by 
+            `data_exploration.finalize_feature_schema()`.
+        continuous_bounds_map (Dict[str, Tuple[float, float]]):
+            A dictionary mapping the *name* of each **continuous** feature
+            to its (min_bound, max_bound) tuple.
+        start_at_zero (bool):
+            - If True, assumes categorical encoding is [0, 1, ..., k-1].
+              Bounds will be set as [-0.5, k - 0.5].
+            - If False, assumes encoding is [1, 2, ..., k].
+              Bounds will be set as [0.5, k + 0.5].
+
+    Returns:
+        Tuple[List[float], List[float]]:
+            A tuple containing two lists: (lower_bounds, upper_bounds).
+
+    Raises:
+        ValueError: If a feature is missing from `continuous_bounds_map`
+                    or if a feature name in the map is not a
+                    continuous feature according to the schema.
+    """
+    # 1. Get feature names and map from schema
+    feature_names = schema.feature_names
+    categorical_index_map = schema.categorical_index_map
+    total_features = len(feature_names)
+
+    if total_features <= 0:
+        _LOGGER.error("Schema contains no features.")
+        raise ValueError()
+        
+    _LOGGER.info(f"Generating bounds for {total_features} total features...")
+
+    # 2. Initialize bound lists
+    lower_bounds: List[Optional[float]] = [None] * total_features
+    upper_bounds: List[Optional[float]] = [None] * total_features
+
+    # 3. Populate categorical bounds (Index-based)
+    if categorical_index_map:
+        for index, cardinality in categorical_index_map.items():
+            if not (0 <= index < total_features):
+                _LOGGER.error(f"Categorical index {index} is out of range for the {total_features} features.")
+                raise ValueError()
+                
+            if start_at_zero:
+                # Rule for [0, k-1]: bounds are [-0.5, k - 0.5]
+                low = -0.5
+                high = float(cardinality) - 0.5
+            else:
+                # Rule for [1, k]: bounds are [0.5, k + 0.5]
+                low = 0.5
+                high = float(cardinality) + 0.5
+                
+            lower_bounds[index] = low
+            upper_bounds[index] = high
+        
+        _LOGGER.info(f"Automatically set bounds for {len(categorical_index_map)} categorical features.")
+    else:
+        _LOGGER.info("No categorical features found in schema.")
+
+    # 4. Populate continuous bounds (Name-based)
+    # Use schema.continuous_feature_names for robust checking
+    continuous_names_set = set(schema.continuous_feature_names)
+    
+    if continuous_names_set != set(continuous_bounds_map.keys()):
+        missing_in_map = continuous_names_set - set(continuous_bounds_map.keys())
+        if missing_in_map:
+            _LOGGER.error(f"The following continuous features are missing from 'continuous_bounds_map': {list(missing_in_map)}")
+        
+        extra_in_map = set(continuous_bounds_map.keys()) - continuous_names_set
+        if extra_in_map:
+            _LOGGER.error(f"The following features in 'continuous_bounds_map' are not defined as continuous in the schema: {list(extra_in_map)}")
+            
+        raise ValueError("Mismatch between 'continuous_bounds_map' and schema's continuous features.")
+
+    count_continuous = 0
+    for name, (low, high) in continuous_bounds_map.items():
+        # Map name to its index in the *feature-only* list
+        # This is guaranteed to be correct by the schema
+        index = feature_names.index(name)
+
+        if lower_bounds[index] is not None:
+            # This should be impossible if schema is correct, but good to check
+            _LOGGER.error(f"Schema conflict: Feature '{name}' (at index {index}) is defined as both continuous and categorical.")
+            raise ValueError()
+
+        lower_bounds[index] = float(low)
+        upper_bounds[index] = float(high)
+        count_continuous += 1
+        
+    _LOGGER.info(f"Manually set bounds for {count_continuous} continuous features.")
+
+    # 5. Final Validation (all Nones should be filled)
+    if None in lower_bounds:
+        missing_indices = [i for i, b in enumerate(lower_bounds) if b is None]
+        missing_names = [feature_names[i] for i in missing_indices]
+        _LOGGER.error(f"Failed to create all bounds. This indicates an internal logic error. Missing: {missing_names}")
+        raise RuntimeError("Internal error: Not all bounds were populated.")
+    
+    # Cast to float lists, as 'None' sentinels are gone
+    return (
+        [float(b) for b in lower_bounds],  # type: ignore
+        [float(b) for b in upper_bounds] # type: ignore
+    )
+
+
 def parse_lower_upper_bounds(source: dict[str,tuple[Any,Any]]):
     """
     Parse lower and upper boundaries, returning 2 lists:
@@ -29,13 +150,16 @@ def parse_lower_upper_bounds(source: dict[str,tuple[Any,Any]]):
     return lower, upper
 
 
-def plot_optimal_feature_distributions(results_dir: Union[str, Path]):
+def plot_optimal_feature_distributions(results_dir: Union[str, Path], verbose: bool=False):
     """
-    Analyzes optimization results and plots the distribution of optimal values for each feature.
+    Analyzes optimization results and plots the distribution of optimal values.
 
-    For features with more than two unique values, this function generates a color-coded 
-    Kernel Density Estimate (KDE) plot. For binary or constant features, it generates a bar plot
-    showing relative frequency.
+    This function is compatible with mixed-type CSVs (strings for
+    categorical features, numbers for continuous). It automatically
+    detects the data type for each feature and generates:
+    
+    - A Bar Plot for categorical (string) features.
+    - A KDE Plot for continuous (numeric) features.
     
     Plots are saved in a subdirectory inside the source directory.
 
@@ -55,10 +179,17 @@ def plot_optimal_feature_distributions(results_dir: Union[str, Path]):
     _LOGGER.info(f"📁 Starting analysis from results in: '{results_dir}'")
     data_to_plot = []
     for df, df_name in yield_dataframes_from_dir(results_path):
+        if df.shape[1] < 2:
+            _LOGGER.warning(f"Skipping '{df_name}': must have at least 2 columns (feature + target).")
+            continue
         melted_df = df.iloc[:, :-1].melt(var_name='feature', value_name='value')
-        melted_df['target'] = df_name.replace("Optimization_", "")
+        melted_df['target'] = df_name
         data_to_plot.append(melted_df)
     
+    if not data_to_plot:
+        _LOGGER.error("No valid data to plot after processing all CSVs.")
+        return
+        
     long_df = pd.concat(data_to_plot, ignore_index=True)
     features = long_df['feature'].unique()
     _LOGGER.info(f"Found data for {len(features)} features across {len(long_df['target'].unique())} targets. Generating plots...")
@@ -66,12 +197,23 @@ def plot_optimal_feature_distributions(results_dir: Union[str, Path]):
     # --- Plotting Loop ---
     for feature_name in features:
         plt.figure(figsize=(12, 7))
+        # Use .copy() to avoid SettingWithCopyWarning
+        # feature_df = long_df[long_df['feature'] == feature_name].copy()
         feature_df = long_df[long_df['feature'] == feature_name]
 
-        # Check if the feature is binary or constant
-        if feature_df['value'].nunique() <= 2:
-            # PLOT 1: For discrete values, calculate percentages and use a true bar plot.
-            # This ensures the X-axis is clean (e.g., just 0 and 1).
+        # --- Type-checking logic ---
+        # Attempt to convert 'value' column to numeric.
+        # errors='coerce' turns non-numeric strings (e.g., 'Category_A') into NaN
+        feature_df['numeric_value'] = pd.to_numeric(feature_df['value'], errors='coerce')
+        
+        # If *any* value failed conversion (is NaN), treat it as categorical.
+        if feature_df['numeric_value'].isna().any():
+            
+            # --- PLOT 1: CATEGORICAL (String-based) ---
+            if verbose:
+                _LOGGER.info(f"Plotting '{feature_name}' as categorical (bar plot).")
+            
+            # Calculate percentages for a clean bar plot
             norm_df = (feature_df.groupby('target')['value']
                        .value_counts(normalize=True)
                        .mul(100)
@@ -79,21 +221,29 @@ def plot_optimal_feature_distributions(results_dir: Union[str, Path]):
                        .reset_index())
             
             ax = sns.barplot(data=norm_df, x='value', y='percent', hue='target')
-            
-            plt.title(f"Optimal Value Distribution for '{feature_name}'", fontsize=16)
             plt.ylabel("Frequency (%)", fontsize=12)
             ax.set_ylim(0, 100) # Set Y-axis from 0 to 100
+            
+            # Rotate x-labels if there are many categories
+            if norm_df['value'].nunique() > 10:
+                plt.xticks(rotation=45, ha='right')
 
         else:
-            # PLOT 2: KDE plot for continuous values.
-            ax = sns.kdeplot(data=feature_df, x='value', hue='target',
+            # --- PLOT 2: CONTINUOUS (Numeric-based) ---
+            # All values were successfully converted to numeric.
+            if verbose:
+                _LOGGER.info(f"Plotting '{feature_name}' as continuous (KDE plot).")
+            
+            # Use the 'numeric_value' column (which is float type) for the KDE
+            ax = sns.kdeplot(data=feature_df, x='numeric_value', hue='target',
                              fill=True, alpha=0.1, warn_singular=False)
-
-            plt.title(f"Optimal Value Distribution for '{feature_name}'", fontsize=16)
-            plt.ylabel("Density", fontsize=12) # Y-axis is "Density" for KDE plots
+            
+            # Set the x-axis label back to the original feature name
+            plt.xlabel("Feature Value", fontsize=12)
+            plt.ylabel("Density", fontsize=12)
 
         # --- Common settings for both plot types ---
-        plt.xlabel("Feature Value", fontsize=12)
+        plt.title(f"Optimal Value Distribution for '{feature_name}'", fontsize=16)
         plt.grid(axis='y', alpha=0.5, linestyle='--')
         
         legend = ax.get_legend()
@@ -106,28 +256,52 @@ def plot_optimal_feature_distributions(results_dir: Union[str, Path]):
         plt.close()
 
     _LOGGER.info(f"All plots saved successfully to: '{output_path}'")
-    
+
 
 def _save_result(
         result_dict: dict,
         save_format: Literal['csv', 'sqlite', 'both'],
         csv_path: Path,
         db_manager: Optional[DatabaseManager] = None,
-        db_table_name: Optional[str] = None
+        db_table_name: Optional[str] = None,
+        categorical_mappings: Optional[Dict[str, Dict[str, int]]] = None
     ):
     """
     Private helper to handle saving a single result to CSV, SQLite, or both.
+    
+    If `categorical_mappings` is provided, it will reverse-map integer values
+    to their string representations before saving.
     """
+    # --- Reverse Mapping Logic ---
+    # Create a copy to hold the values to be saved
+    save_dict = result_dict.copy()
+    
+    if categorical_mappings:
+        for feature_name, mapping in categorical_mappings.items():
+            if feature_name in save_dict:
+                # Create a reverse map {0: 'Category_A', 1: 'Category_B'}
+                reverse_map = {idx: name for name, idx in mapping.items()}
+                
+                # Get the integer value from the results (e.g., 0)
+                int_value = save_dict[feature_name]
+                
+                # Find the corresponding string (e.g., 'Category_A')
+                # Use .get() for safety, defaulting to the original value if not found
+                string_value = reverse_map.get(int_value, int_value)
+                
+                # Update the dictionary that will be saved
+                save_dict[feature_name] = string_value
+    
     # Save to CSV
     if save_format in ['csv', 'both']:
-        df_row = pd.DataFrame([result_dict])
+        df_row = pd.DataFrame([save_dict])
         file_exists = csv_path.exists()
         df_row.to_csv(csv_path, mode='a', index=False, header=not file_exists)
 
     # Save to SQLite
     if save_format in ['sqlite', 'both']:
         if db_manager and db_table_name:
-            db_manager.insert_row(db_table_name, result_dict)
+            db_manager.insert_row(db_table_name, save_dict)
         else:
             _LOGGER.warning("SQLite saving requested but db_manager or table_name not provided.")
 

ml_tools/path_manager.py

@@ -2,9 +2,10 @@ from pprint import pprint
 from typing import Optional, List, Dict, Union, Literal
 from pathlib import Path
 import re
+import sys
+
 from ._script_info import _script_info
 from ._logger import _LOGGER
-import sys
 
 
 __all__ = [
@@ -13,6 +14,7 @@ __all__ = [
     "sanitize_filename",
     "list_csv_paths",
     "list_files_by_extension",
+    "list_subdirectories"
 ]
 
 
@@ -20,15 +22,35 @@ class PathManager:
     """
     Manages and stores a project's file paths, acting as a centralized
     "path database". It supports both development mode and applications
-    bundled with Pyinstaller.
+    bundled with Pyinstaller or Nuitka.
     
-    Supports python dictionary syntax.
+    All keys provided to the manager are automatically sanitized to ensure
+    they are valid Python identifiers. This allows for clean, attribute-style
+    access. The sanitization process involves replacing whitespace with
+    underscores and removing special characters.
     """
     def __init__(
         self,
         anchor_file: str,
         base_directories: Optional[List[str]] = None
     ):
+        """
+        Sets up the core paths for a project by anchoring to a specific file.
+
+        The manager automatically registers a 'ROOT' path, which points to the
+        root of the package, and can pre-register common subdirectories found
+        directly within that root.
+
+        Args:
+            anchor_file (str): The path to a file within your package, typically
+                            the `__file__` of the script where PathManager
+                            is instantiated. This is used to locate the
+                            package root directory.
+            base_directories (List[str] | None): An optional list of strings,
+                                                    where each string is the name
+                                                    of a subdirectory to register
+                                                    relative to the package root.
+        """
         resolved_anchor_path = Path(anchor_file).resolve()
         self._package_name = resolved_anchor_path.parent.name
         self._is_bundled, bundle_root = self._get_bundle_root()
@@ -42,13 +64,17 @@ class PathManager:
             package_root = resolved_anchor_path.parent
 
         # Register the root of the package itself
-        self._paths["ROOT"] = package_root
+        self.ROOT = package_root
 
         # Register all the base directories
         if base_directories:
             for dir_name in base_directories:
-                # This logic works for both dev mode and bundled mode
-                self._paths[dir_name] = package_root / dir_name
+                sanitized_dir_name = self._sanitize_key(dir_name)
+                self._check_underscore_key(sanitized_dir_name)
+                setattr(self, sanitized_dir_name, package_root / sanitized_dir_name)
+        
+        # Signal that initialization is complete.
+        self._initialized = True
     
     def _get_bundle_root(self) -> tuple[bool, Optional[str]]:
         """
@@ -71,47 +97,35 @@ class PathManager:
         # --- Not Bundled ---
         else:
             return False, None
+        
+    def _check_underscore_key(self, key: str) -> None:
+        if key.startswith("_"):
+            _LOGGER.error(f"Path key '{key}' cannot start with underscores.")
+            raise ValueError()
 
-    def get(self, key: str) -> Path:
-        """
-        Retrieves a stored path by its key.
-
-        Args:
-            key (str): The key of the path to retrieve.
-
-        Returns:
-            Path: The resolved, absolute Path object.
-
-        Raises:
-            KeyError: If the key is not found in the manager.
-        """
-        try:
-            return self._paths[key]
-        except KeyError:
-            _LOGGER.error(f"Path key '{key}' not found.")
-            raise
-
-    def update(self, new_paths: Dict[str, Union[str, Path]], overwrite: bool = False) -> None:
+    def update(self, new_paths: Dict[str, Union[str, Path]]) -> None:
         """
-        Adds new paths or overwrites existing ones in the manager.
+        Adds new paths in the manager.
 
         Args:
             new_paths (Dict[str, Union[str, Path]]): A dictionary where keys are
                                     the identifiers and values are the
-                                    Path objects or strings to store.
-            overwrite (bool): If False (default), raises a KeyError if any
-                            key in new_paths already exists. If True,
-                            allows overwriting existing keys.
+                                    Path objects to store.
         """
-        if not overwrite:
-            for key in new_paths:
-                if key in self._paths:
-                    _LOGGER.error(f"Path key '{key}' already exists in the manager. To replace it, call update() with overwrite=True.")
-                    raise KeyError
-
-        # Resolve any string paths to Path objects before storing
-        resolved_new_paths = {k: Path(v) for k, v in new_paths.items()}
-        self._paths.update(resolved_new_paths)
+        # Pre-check
+        for key in new_paths:
+            sanitized_key = self._sanitize_key(key)
+            self._check_underscore_key(sanitized_key)
+            if hasattr(self, sanitized_key):
+                _LOGGER.error(f"Cannot add path for key '{sanitized_key}' ({key}): an attribute with this name already exists.")
+                raise KeyError()
+        
+        # If no conflicts, add new paths
+        for key, value in new_paths.items():
+            self.__setattr__(key, value)
+        
+    def _sanitize_key(self, key: str):
+        return sanitize_filename(key)
         
     def make_dirs(self, keys: Optional[List[str]] = None, verbose: bool = False) -> None:
         """
@@ -146,7 +160,7 @@ class PathManager:
             if path.suffix:  # It's a file, not a directory
                 continue
 
-            # --- THE CRITICAL CHECK ---
+            # --- CRITICAL CHECK ---
             # Determine if the path is inside the main application package.
             is_internal_path = package_root and path.is_relative_to(package_root)
 
@@ -185,15 +199,20 @@ class PathManager:
     # --- Dictionary-Style Methods ---
     def __getitem__(self, key: str) -> Path:
         """Allows dictionary-style getting, e.g., PM['my_key']"""
-        return self.get(key)
+        return self.__getattr__(key)
 
     def __setitem__(self, key: str, value: Union[str, Path]):
-        """Allows dictionary-style setting, does not allow overwriting, e.g., PM['my_key'] = path"""
-        self.update({key: value}, overwrite=False)
+        """Allows dictionary-style setting, e.g., PM['my_key'] = path"""
+        sanitized_key = self._sanitize_key(key)
+        self._check_underscore_key(sanitized_key)
+        self.__setattr__(sanitized_key, value)
 
     def __contains__(self, key: str) -> bool:
         """Allows checking for a key's existence, e.g., if 'my_key' in PM"""
-        return key in self._paths
+        sanitized_key = self._sanitize_key(key)
+        true_false = sanitized_key in self._paths
+        # print(f"key {sanitized_key} in current path dictionary keys: {true_false}")
+        return true_false
 
     def __len__(self) -> int:
         """Allows getting the number of paths, e.g., len(PM)"""
@@ -210,6 +229,54 @@ class PathManager:
     def items(self):
         """Returns all registered (key, Path) pairs."""
         return self._paths.items()
+    
+    def __getattr__(self, name: str) -> Path:
+        """
+        Allows attribute-style access to paths, e.g., PM.data.
+        """
+        # Block access to private attributes
+        if name.startswith('_'):
+            _LOGGER.error(f"Access to private attribute '{name}' is not allowed, remove leading underscore.")
+            raise AttributeError()
+        
+        sanitized_name = self._sanitize_key(name)
+        
+        try:
+            # Look for the key in our internal dictionary
+            return self._paths[sanitized_name]
+        except KeyError:
+            # If not found, raise an AttributeError
+            _LOGGER.error(f"'{type(self).__name__}' object has no attribute or path key '{sanitized_name}'")
+            raise AttributeError()
+    
+    def __setattr__(self, name: str, value: Union[str, Path, bool, dict, str, int, tuple]):
+        """Allows attribute-style setting of paths, e.g., PM.data = 'path/to/data'."""
+        # Check for internal attributes, which are set directly on the object.
+        if name.startswith('_'):
+            # This check prevents setting new private attributes after __init__ is done.
+            is_initialized = self.__dict__.get('_initialized', False)
+            if is_initialized:
+                _LOGGER.error(f"Cannot set private attribute '{name}' after initialization.")
+                raise AttributeError()
+            super().__setattr__(name, value)
+            return
+
+        # Sanitize the key for the public path.
+        sanitized_name = self._sanitize_key(name)
+        self._check_underscore_key(sanitized_name)
+
+        # Prevent overwriting existing methods (e.g., PM.status = 'foo').
+        # This check looks at the class, not the instance therefore won't trigger __getattr__.
+        if hasattr(self.__class__, sanitized_name):
+            _LOGGER.error(f"Cannot overwrite existing attribute or method '{sanitized_name}' ({name}).")
+            raise AttributeError()
+        
+        if not isinstance(value, (str, Path)):
+            _LOGGER.error(f"Cannot assign type '{type(value).__name__}' to a path. Must be str or Path.")
+            raise TypeError
+
+        # If all checks pass, treat it as a public path and store it in the _paths dictionary.
+        self._paths[sanitized_name] = Path(value)
 
 
 def make_fullpath(
@@ -385,5 +452,37 @@ def list_files_by_extension(directory: Union[str,Path], extension: str, verbose:
     return name_path_dict
 
 
+def list_subdirectories(root_dir: Union[str,Path], verbose: bool=True) -> dict[str, Path]:
+    """
+    Scans a directory and returns a dictionary of its immediate subdirectories.
+
+    Args:
+        root_dir (str | Path): The path to the directory to scan.
+        verbose (bool): If True, prints the number of directories found. 
+
+    Returns:
+        dict[str, Path]: A dictionary mapping subdirectory names (str) to their full Path objects.
+    """
+    root_path = make_fullpath(root_dir, enforce="directory")
+    
+    directories = [p.resolve() for p in root_path.iterdir() if p.is_dir()]
+    
+    if len(directories) < 1:
+        _LOGGER.error(f"No subdirectories found inside '{root_path}'")
+        raise IOError()
+    
+    if verbose:
+        count = len(directories)
+        # Use pluralization for better readability
+        plural = 'ies' if count != 1 else 'y'
+        print(f"Found {count} subdirector{plural} in '{root_path.name}'.")
+    
+    # Create a dictionary where the key is the directory's name (a string)
+    # and the value is the full Path object.
+    dir_map = {p.name: p for p in directories}
+    
+    return dir_map
+
+
 def info():
     _script_info(__all__)