dragon-ml-toolbox 14.3.0__py3-none-any.whl → 14.8.0__py3-none-any.whl

ml_tools/ML_vision_transformers.py

@@ -1,14 +1,18 @@
-from typing import Union, Dict, Type, Callable
+from typing import Union, Dict, Type, Callable, Optional, Any, List, Literal
 from PIL import ImageOps, Image
+from torchvision import transforms
+from pathlib import Path
 
 from ._logger import _LOGGER
 from ._script_info import _script_info
 from .keys import VisionTransformRecipeKeys
+from .path_manager import make_fullpath
 
 
 __all__ = [
     "TRANSFORM_REGISTRY",
-    "ResizeAspectFill"
+    "ResizeAspectFill",
+    "create_offline_augmentations"
 ]
 
 # --- Custom Vision Transform Class ---
@@ -23,9 +27,8 @@ class ResizeAspectFill:
     """
     def __init__(self, pad_color: Union[str, int] = "black") -> None:
         self.pad_color = pad_color
-        # Store kwargs to allow for recreation
+        # Store kwargs to allow for re-creation
         self.__setattr__(VisionTransformRecipeKeys.KWARGS, {"pad_color": pad_color})
-        # self._kwargs = {"pad_color": pad_color}
 
     def __call__(self, image: Image.Image) -> Image.Image:
         if not isinstance(image, Image.Image):
@@ -47,12 +50,154 @@ class ResizeAspectFill:
             padding = (left_padding, 0, right_padding, 0)
 
         return ImageOps.expand(image, padding, fill=self.pad_color)
-    
 
-#NOTE: Add custom transforms here.
+
+#NOTE: Add custom transforms.
 TRANSFORM_REGISTRY: Dict[str, Type[Callable]] = {
     "ResizeAspectFill": ResizeAspectFill,
 }
 
+
+def _build_transform_from_recipe(recipe: Dict[str, Any]) -> transforms.Compose:
+    """Internal helper to build a transform pipeline from a recipe dict."""
+    pipeline_steps: List[Callable] = []
+    
+    if VisionTransformRecipeKeys.PIPELINE not in recipe:
+        _LOGGER.error("Recipe dict is invalid: missing 'pipeline' key.")
+        raise ValueError("Invalid recipe format.")
+
+    for step in recipe[VisionTransformRecipeKeys.PIPELINE]:
+        t_name = step.get(VisionTransformRecipeKeys.NAME)
+        t_kwargs = step.get(VisionTransformRecipeKeys.KWARGS, {})
+        
+        if not t_name:
+            _LOGGER.error(f"Invalid transform step, missing 'name': {step}")
+            continue
+        
+        transform_class: Any = None
+
+        # 1. Check standard torchvision transforms
+        if hasattr(transforms, t_name):
+            transform_class = getattr(transforms, t_name)
+        # 2. Check custom transforms
+        elif t_name in TRANSFORM_REGISTRY:
+            transform_class = TRANSFORM_REGISTRY[t_name]
+        # 3. Not found
+        else:
+            _LOGGER.error(f"Unknown transform '{t_name}' in recipe. Not found in torchvision.transforms or TRANSFORM_REGISTRY.")
+            raise ValueError(f"Unknown transform name: {t_name}")
+            
+        # Instantiate the transform
+        try:
+            pipeline_steps.append(transform_class(**t_kwargs))
+        except Exception as e:
+            _LOGGER.error(f"Failed to instantiate transform '{t_name}' with kwargs {t_kwargs}: {e}")
+            raise
+            
+    return transforms.Compose(pipeline_steps)
+
+
+def create_offline_augmentations(
+    input_directory: Union[str, Path],
+    output_directory: Union[str, Path],
+    results_per_image: int,
+    recipe: Optional[Dict[str, Any]] = None,
+    save_format: Literal["WEBP", "JPEG", "PNG", "BMP", "TIF"] = "WEBP",
+    save_quality: int = 80
+) -> None:
+    """
+    Reads all valid images from an input directory, applies augmentations,
+    and saves the new images to an output directory (offline augmentation).
+
+    Skips subdirectories in the input path.
+
+    Args:
+        input_directory (Union[str, Path]): Path to the directory of source images.
+        output_directory (Union[str, Path]): Path to save the augmented images.
+        results_per_image (int): The number of augmented versions to create
+                                 for each source image.
+        recipe (Optional[Dict[str, Any]]): A transform recipe dictionary. If None,
+                                           a default set of strong, random
+                                           augmentations will be used.
+        save_format (str): The format to save images (e.g., "WEBP", "JPEG", "PNG").
+                           Defaults to "WEBP" for good compression.
+        save_quality (int): The quality for lossy formats (1-100). Defaults to 80.
+    """
+    VALID_IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.bmp', '.webp', '.tif', '.tiff')
+    
+    # --- 1. Validate Paths ---
+    in_path = make_fullpath(input_directory, enforce="directory")
+    out_path = make_fullpath(output_directory, make=True, enforce="directory")
+    
+    _LOGGER.info(f"Starting offline augmentation:\n\tInput: {in_path}\n\tOutput: {out_path}")
+
+    # --- 2. Find Images ---
+    image_files = [
+        f for f in in_path.iterdir()
+        if f.is_file() and f.suffix.lower() in VALID_IMG_EXTENSIONS
+    ]
+    
+    if not image_files:
+        _LOGGER.warning(f"No valid image files found in {in_path}.")
+        return
+
+    _LOGGER.info(f"Found {len(image_files)} images to process.")
+
+    # --- 3. Define Transform Pipeline ---
+    transform_pipeline: transforms.Compose
+    
+    if recipe:
+        _LOGGER.info("Building transformations from provided recipe.")
+        try:
+            transform_pipeline = _build_transform_from_recipe(recipe)
+        except Exception as e:
+            _LOGGER.error(f"Failed to build transform from recipe: {e}")
+            return
+    else:
+        _LOGGER.info("No recipe provided. Using default random augmentation pipeline.")
+        # Default "random" pipeline
+        transform_pipeline = transforms.Compose([
+            transforms.RandomResizedCrop(256, scale=(0.4, 1.0)),
+            transforms.RandomHorizontalFlip(p=0.5),
+            transforms.RandomRotation(degrees=90),
+            transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.15),
+            transforms.RandomPerspective(distortion_scale=0.2, p=0.4),
+            transforms.RandomAffine(degrees=0, translate=(0.1, 0.1)),
+            transforms.RandomApply([
+                transforms.GaussianBlur(kernel_size=3)
+            ], p=0.3)
+        ])
+
+    # --- 4. Process Images ---
+    total_saved = 0
+    format_upper = save_format.upper()
+    
+    for img_path in image_files:
+        _LOGGER.debug(f"Processing {img_path.name}...")
+        try:
+            original_image = Image.open(img_path).convert("RGB")
+            
+            for i in range(results_per_image):
+                new_stem = f"{img_path.stem}_aug_{i+1:03d}"
+                output_path = out_path / f"{new_stem}.{format_upper.lower()}"
+                
+                # Apply transform
+                transformed_image = transform_pipeline(original_image)
+                
+                # Save
+                transformed_image.save(
+                    output_path, 
+                    format=format_upper, 
+                    quality=save_quality,
+                    optimize=True # Add optimize flag
+                )
+                total_saved += 1
+                
+        except Exception as e:
+            _LOGGER.warning(f"Failed to process or save augmentations for {img_path.name}: {e}")
+
+    _LOGGER.info(f"Offline augmentation complete. Saved {total_saved} new images.")
+
+
 def info():
     _script_info(__all__)

ml_tools/ensemble_evaluation.py

@@ -112,7 +112,7 @@ def evaluate_model_classification(
         report_df = pd.DataFrame(report_dict).iloc[:-1, :].T
         plt.figure(figsize=figsize)
         sns.heatmap(report_df, annot=True, cmap=heatmap_cmap, fmt='.2f', 
-                    annot_kws={"size": base_fontsize - 4})
+                    annot_kws={"size": base_fontsize - 4}, vmin=0.0, vmax=1.0)
         plt.title(f"{model_name} - {target_name}", fontsize=base_fontsize)
         plt.xticks(fontsize=base_fontsize - 2)
         plt.yticks(fontsize=base_fontsize - 2)
@@ -133,6 +133,7 @@ def evaluate_model_classification(
         normalize="true",
         ax=ax
     )
+    disp.im_.set_clim(vmin=0.0, vmax=1.0)
 
     ax.set_title(f"{model_name} - {target_name}", fontsize=base_fontsize)
     ax.tick_params(axis='both', labelsize=base_fontsize)
@@ -327,7 +328,8 @@ def plot_calibration_curve(
     target_name: str,
     figure_size: tuple = (10, 10),
     base_fontsize: int = 24,
-    n_bins: int = 15
+    n_bins: int = 15,
+    line_color: str = 'darkorange'
 ) -> plt.Figure: # type: ignore
     """
     Plots the calibration curve (reliability diagram) for a classifier.
@@ -348,22 +350,63 @@ def plot_calibration_curve(
     """
     fig, ax = plt.subplots(figsize=figure_size)
     
-    disp = CalibrationDisplay.from_estimator(
-        model,
-        x_test,
-        y_test,
-        n_bins=n_bins,
-        ax=ax
+    # --- Step 1: Get probabilities from the estimator ---
+    # We do this manually so we can pass them to from_predictions
+    try:
+        y_prob = model.predict_proba(x_test)
+        # Use probabilities for the positive class (assuming binary)
+        y_score = y_prob[:, 1]
+    except Exception as e:
+        _LOGGER.error(f"Could not get probabilities from model: {e}")
+        plt.close(fig)
+        return fig # Return empty figure
+
+    # --- Step 2: Get binned data *without* plotting ---
+    with plt.ioff(): 
+        fig_temp, ax_temp = plt.subplots()
+        cal_display_temp = CalibrationDisplay.from_predictions(
+            y_test, 
+            y_score, 
+            n_bins=n_bins, 
+            ax=ax_temp,
+            name="temp"
+        )
+        line_x, line_y = cal_display_temp.line_.get_data() # type: ignore
+        plt.close(fig_temp)
+
+    # --- Step 3: Build the plot from scratch on ax ---
+
+    # 3a. Plot the ideal diagonal line
+    ax.plot([0, 1], [0, 1], 'k--', label='Perfectly calibrated')
+
+    # 3b. Use regplot for the regression line and its CI
+    sns.regplot(
+        x=line_x, 
+        y=line_y,
+        ax=ax,
+        scatter=False,  # No scatter dots
+        label=f"Calibration Curve ({n_bins} bins)",
+        line_kws={
+            'color': line_color,
+            'linestyle': '--', 
+            'linewidth': 2
+        }
     )
 
+    # --- Step 4: Apply original formatting ---
     ax.set_title(f"{model_name} - Reliability Curve for {target_name}", fontsize=base_fontsize)
     ax.tick_params(axis='both', labelsize=base_fontsize - 2)
     ax.set_xlabel("Mean Predicted Probability", fontsize=base_fontsize)
     ax.set_ylabel("Fraction of Positives", fontsize=base_fontsize)
-    ax.legend(fontsize=base_fontsize - 4)
+    
+    # Set limits
+    ax.set_ylim(0.0, 1.0)
+    ax.set_xlim(0.0, 1.0)
+    
+    ax.legend(fontsize=base_fontsize - 4, loc='lower right')
     fig.tight_layout()
 
-    # Save figure
+    # --- Step 5: Save figure (using original logic) ---
     save_path = make_fullpath(save_dir, make=True)
     sanitized_target_name = sanitize_filename(target_name)
     full_save_path = save_path / f"Calibration_Plot_{sanitized_target_name}.svg"

ml_tools/keys.py

@@ -104,8 +104,9 @@ class VisionTransformRecipeKeys:
     TASK = "task"
     PIPELINE = "pipeline"
     NAME = "name"
-    KWARGS = "_kwargs"
+    KWARGS = "kwargs"
     PRE_TRANSFORMS = "pre_transforms"
+    
     RESIZE_SIZE = "resize_size"
     CROP_SIZE = "crop_size"
     MEAN = "mean"