autogluon.multimodal 1.2.1b20250303__py3-none-any.whl → 1.2.1b20250305__py3-none-any.whl

autogluon/multimodal/data/process_image.py

@@ -1,4 +1,7 @@
+import ast
+import copy
 import logging
+import random
 import warnings
 from io import BytesIO
 from typing import Callable, Dict, List, Optional, Union
@@ -6,24 +9,25 @@ from typing import Callable, Dict, List, Optional, Union
 import numpy as np
 import PIL
 import torch
+from omegaconf import ListConfig
 from PIL import ImageFile
 from torch import nn
 from torchvision import transforms
 
-from .utils import construct_image_processor, image_mean_std
+from .randaug import RandAugment
+from .trivial_augmenter import TrivialAugment
 
 try:
     from torchvision.transforms import InterpolationMode
 
     BICUBIC = InterpolationMode.BICUBIC
+    NEAREST = InterpolationMode.NEAREST
 except ImportError:
     BICUBIC = PIL.Image.BICUBIC
+    NEAREST = PIL.Image.NEAREST
 
-from ..constants import CLIP, COLUMN, IMAGE, IMAGE_BASE64_STR, IMAGE_BYTEARRAY, IMAGE_VALID_NUM, TIMM_IMAGE
-from ..models.clip import CLIPForImageText
-from ..models.timm_image import TimmAutoModelForImagePrediction
+from ..constants import COLUMN, IMAGE, IMAGE_BASE64_STR, IMAGE_BYTEARRAY, IMAGE_VALID_NUM
 from .collator import PadCollator, StackCollator
-from .utils import extract_value_from_config
 
 logger = logging.getLogger(__name__)
 ImageFile.LOAD_TRUNCATED_IMAGES = True
@@ -40,11 +44,10 @@ class ImageProcessor:
         model: nn.Module,
         train_transforms: Union[List[str], Callable, List[Callable]],
         val_transforms: Union[List[str], Callable, List[Callable]],
-        norm_type: Optional[str] = None,
-        size: Optional[int] = None,
-        max_img_num_per_col: Optional[int] = 1,
+        max_image_num_per_column: Optional[int] = 1,
         missing_value_strategy: Optional[str] = "zero",
-        requires_column_info: bool = False,
+        requires_column_info: Optional[bool] = False,
+        dropout: Optional[float] = 0,
     ):
         """
         Parameters
@@ -55,18 +58,7 @@ class ImageProcessor:
             A list of image transforms used in training. Note that the transform order matters.
         val_transforms
             A list of image transforms used in validation/test/prediction. Note that the transform order matters.
-        norm_type
-            How to normalize an image. We now support:
-            - inception
-                Normalize image by IMAGENET_INCEPTION_MEAN and IMAGENET_INCEPTION_STD from timm
-            - imagenet
-                Normalize image by IMAGENET_DEFAULT_MEAN and IMAGENET_DEFAULT_STD from timm
-            - clip
-                Normalize image by mean (0.48145466, 0.4578275, 0.40821073) and
-                std (0.26862954, 0.26130258, 0.27577711), used for CLIP.
-        size
-            The provided width / height of a square image.
-        max_img_num_per_col
+        max_image_num_per_column
             The maximum number of images one sample can have.
         missing_value_strategy
             How to deal with a missing image. We now support:
@@ -77,6 +69,7 @@ class ImageProcessor:
         requires_column_info
             Whether to require feature column information in dataloader.
         """
+        logger.debug(f"initializing image processor for model {model.prefix}")
         self.train_transforms = train_transforms
         self.val_transforms = val_transforms
         logger.debug(f"image training transforms: {self.train_transforms}")
@@ -85,62 +78,25 @@ class ImageProcessor:
         self.prefix = model.prefix
         self.missing_value_strategy = missing_value_strategy
         self.requires_column_info = requires_column_info
-        self.size = None
-        self.mean = None
-        self.std = None
-        if isinstance(model, CLIPForImageText):
-            config = model.model.vision_model.config
-        else:
-            config = model.config
-
-        if model is not None:
-            self.size, self.mean, self.std = self.extract_default(config)
-            if isinstance(model, TimmAutoModelForImagePrediction):
-                if model.support_variable_input_size() and size is not None:
-                    # We have detected that the model supports using an image size that is
-                    # different from the pretrained model, e.g., ConvNets with global pooling
-                    if size < self.size:
-                        logger.warning(
-                            f"The provided image size={size} is smaller than the default size "
-                            f"of the pretrained backbone, which is {self.size}. "
-                            f"Detailed configuration of the backbone is in {config}. "
-                            f"You may like to double check your configuration."
-                        )
-                    self.size = size
-            elif size is not None and size != self.size:
-                logger.warning(
-                    f"The model does not support using an image size that is different from the default size. "
-                    f"Provided image size={size}. Default size={self.size}. "
-                    f"Detailed model configuration={config}. We have ignored the provided image size."
-                )
-        if self.size is None:
-            if size is not None:
-                self.size = size
-                logger.debug(f"using provided image size: {self.size}")
-            else:
-                raise ValueError("image size is missing")
-        else:
-            logger.debug(f"using detected image size: {self.size}")
-        if self.mean is None or self.std is None:
-            if norm_type is not None:
-                self.mean, self.std = image_mean_std(norm_type)
-                logger.debug(f"using provided normalization: {norm_type}")
-            else:
-                raise ValueError("image normalization mean and std are missing")
-        else:
-            logger.debug(f"using detected image normalization: {self.mean} and {self.std}")
+        assert 0 <= dropout <= 1
+        if dropout > 0:
+            logger.debug(f"image dropout probability: {dropout}")
+        self.dropout = dropout
+        self.size = model.image_size
+        self.mean = model.image_mean
+        self.std = model.image_std
+
         self.normalization = transforms.Normalize(self.mean, self.std)
-        self.max_img_num_per_col = max_img_num_per_col
-        if max_img_num_per_col <= 0:
-            logger.debug(f"max_img_num_per_col {max_img_num_per_col} is reset to 1")
-            max_img_num_per_col = 1
-        self.max_img_num_per_col = max_img_num_per_col
-        logger.debug(f"max_img_num_per_col: {max_img_num_per_col}")
-
-        self.train_processor = construct_image_processor(
+        if max_image_num_per_column <= 0:
+            logger.debug(f"max_image_num_per_column {max_image_num_per_column} is reset to 1")
+            max_image_num_per_column = 1
+        self.max_image_num_per_column = max_image_num_per_column
+        logger.debug(f"max_image_num_per_column: {max_image_num_per_column}")
+
+        self.train_processor = self.construct_image_processor(
             image_transforms=self.train_transforms, size=self.size, normalization=self.normalization
         )
-        self.val_processor = construct_image_processor(
+        self.val_processor = self.construct_image_processor(
             image_transforms=self.val_transforms, size=self.size, normalization=self.normalization
         )
 
@@ -159,7 +115,7 @@ class ImageProcessor:
     def collate_fn(self, image_column_names: Optional[List] = None, per_gpu_batch_size: Optional[int] = None) -> Dict:
         """
         Collate images into a batch. Here it pads images since the image number may
-        vary from sample to sample. Samples with less images will be padded zeros.
+        vary from sample to sample. Samples with fewer images will be padded zeros.
         The valid image numbers of samples will be stacked into a vector.
         This function will be used when creating Pytorch DataLoader.
 
@@ -182,65 +138,23 @@ class ImageProcessor:
 
         return fn
 
-    def extract_default(self, config=None):
-        """
-        Extract some default hyper-parameters, e.g., image size, mean, and std,
-        from a pre-trained (timm or huggingface) checkpoint.
-
-        Parameters
-        ----------
-        config
-            Config of a pre-trained checkpoint.
-
-        Returns
-        -------
-        image_size
-            Image width/height.
-        mean
-            Image normalization mean.
-        std
-            Image normalizaiton std.
-        """
-        if self.prefix.lower().startswith(TIMM_IMAGE):
-            image_size = config["input_size"][-1]
-            mean = config["mean"]
-            std = config["std"]
-        elif self.prefix.lower().startswith(CLIP):
-            extracted = extract_value_from_config(
-                config=config.to_diff_dict(),
-                keys=("image_size",),
-            )
-            if len(extracted) == 0:
-                image_size = None
-            elif len(extracted) >= 1:
-                image_size = extracted[0]
-                if isinstance(image_size, tuple):
-                    image_size = image_size[-1]
-            else:
-                raise ValueError(f" more than one image_size values are detected: {extracted}")
-            mean = None
-            std = None
-        else:
-            raise ValueError(f"Unknown image processor prefix: {self.prefix}")
-        return image_size, mean, std
-
     def process_one_sample(
         self,
-        image_features: Dict[str, Union[List[str], List[bytearray]]],
-        feature_modalities: Dict[str, List[str]],
+        images: Dict[str, Union[List[str], List[bytearray]]],
+        sub_dtypes: Dict[str, str],
         is_training: bool,
         image_mode: Optional[str] = "RGB",
     ) -> Dict:
         """
         Read images, process them, and stack them. One sample can have multiple images,
-        resulting in a tensor of (n, 3, size, size), where n <= max_img_num_per_col is the available image number.
+        resulting in a tensor of (n, 3, size, size), where n <= max_image_num_per_column is the available image number.
 
         Parameters
         ----------
-        image_features
+        images
             One sample may have multiple image columns in a pd.DataFrame and multiple images
             inside each image column.
-        feature_modalities
+        sub_dtypes
             What modality each column belongs to.
         is_training
             Whether to process images in the training mode.
@@ -252,35 +166,38 @@ class ImageProcessor:
         -------
         A dictionary containing one sample's images and their number.
         """
-        images = []
+        valid_images = []
         zero_images = []
         ret = {}
         column_start = 0
 
-        for per_col_name, per_col_image_features in image_features.items():
-            for img_feature in per_col_image_features[: self.max_img_num_per_col]:
-                with warnings.catch_warnings():
-                    warnings.filterwarnings(
-                        "ignore",
-                        message=(
-                            "Palette images with Transparency expressed in bytes should be converted to RGBA images"
-                        ),
-                    )
-                    is_zero_img = False
-                    try:
-                        if feature_modalities.get(per_col_name) in [IMAGE_BYTEARRAY, IMAGE_BASE64_STR]:
-                            image_feature = BytesIO(img_feature)
-                        else:
-                            image_feature = img_feature
-                        with PIL.Image.open(image_feature) as img:
-                            img = img.convert(image_mode)
-                    except Exception as e:
-                        if self.missing_value_strategy.lower() == "zero":
-                            logger.debug(f"Using a zero image due to '{e}'")
-                            img = PIL.Image.new(image_mode, (self.size, self.size), color=0)
-                            is_zero_img = True
-                        else:
-                            raise e
+        for per_col_name, per_col_image_raw in images.items():
+            for img_raw in per_col_image_raw[: self.max_image_num_per_column]:
+                if is_training and self.dropout > 0 and random.uniform(0, 1) <= self.dropout:
+                    img = PIL.Image.new(image_mode, (self.size, self.size), color=0)
+                    is_zero_img = True
+                else:
+                    with warnings.catch_warnings():
+                        warnings.filterwarnings(
+                            "ignore",
+                            message=(
+                                "Palette images with Transparency expressed in bytes should be converted to RGBA images"
+                            ),
+                        )
+                        is_zero_img = False
+                        try:
+                            if sub_dtypes.get(per_col_name) in [IMAGE_BYTEARRAY, IMAGE_BASE64_STR]:
+                                img_raw = BytesIO(img_raw)
+
+                            with PIL.Image.open(img_raw) as img:
+                                img = img.convert(image_mode)
+                        except Exception as e:
+                            if self.missing_value_strategy.lower() == "zero":
+                                logger.debug(f"Using a zero image due to '{e}'")
+                                img = PIL.Image.new(image_mode, (self.size, self.size), color=0)
+                                is_zero_img = True
+                            else:
+                                raise e
                 if is_training:
                     img = self.train_processor(img)
                 else:
@@ -289,29 +206,152 @@ class ImageProcessor:
                 if is_zero_img:
                     zero_images.append(img)
                 else:
-                    images.append(img)
+                    valid_images.append(img)
 
             if self.requires_column_info:
                 # only count the valid images since they are put ahead of the zero images in the below returning
                 ret[f"{self.image_column_prefix}_{per_col_name}"] = np.array(
-                    [column_start, len(images)], dtype=np.int64
+                    [column_start, len(valid_images)], dtype=np.int64
                 )
-                column_start = len(images)
+                column_start = len(valid_images)
 
         ret.update(
             {
                 self.image_key: torch.tensor([])
-                if len(images + zero_images) == 0
-                else torch.stack(images + zero_images, dim=0),
-                self.image_valid_num_key: len(images),
+                if len(valid_images + zero_images) == 0
+                else torch.stack(valid_images + zero_images, dim=0),
+                self.image_valid_num_key: len(valid_images),
             }
         )
         return ret
 
+    @staticmethod
+    def get_image_transform_funcs(transform_types: Union[List[str], ListConfig, List[Callable]], size: int):
+        """
+        Parse a list of transform strings into callable objects.
+
+        Parameters
+        ----------
+        transform_types
+            A list of transforms, which can be strings or callable objects.
+        size
+            Image size.
+
+        Returns
+        -------
+        A list of transform objects.
+        """
+        image_transforms = []
+
+        if not transform_types:
+            return image_transforms
+
+        if isinstance(transform_types, ListConfig):
+            transform_types = list(transform_types)
+        elif not isinstance(transform_types, list):
+            transform_types = [transform_types]
+
+        if all([isinstance(trans_type, str) for trans_type in transform_types]):
+            pass
+        elif all([isinstance(trans_type, Callable) for trans_type in transform_types]):
+            return copy.copy(transform_types)
+        else:
+            raise ValueError(
+                f"transform_types {transform_types} contain neither all strings nor all callable objects."
+            )
+
+        for trans_type in transform_types:
+            args = None
+            kargs = None
+            if "(" in trans_type:
+                trans_mode = trans_type[0 : trans_type.find("(")]
+                if "{" in trans_type:
+                    kargs = ast.literal_eval(trans_type[trans_type.find("{") : trans_type.rfind(")")])
+                else:
+                    args = ast.literal_eval(trans_type[trans_type.find("(") :])
+            else:
+                trans_mode = trans_type
+
+            if trans_mode == "resize_to_square":
+                image_transforms.append(transforms.Resize((size, size), interpolation=BICUBIC))
+            elif trans_mode == "resize_gt_to_square":
+                image_transforms.append(transforms.Resize((size, size), interpolation=NEAREST))
+            elif trans_mode == "resize_shorter_side":
+                image_transforms.append(transforms.Resize(size, interpolation=BICUBIC))
+            elif trans_mode == "center_crop":
+                image_transforms.append(transforms.CenterCrop(size))
+            elif trans_mode == "random_resize_crop":
+                image_transforms.append(transforms.RandomResizedCrop(size))
+            elif trans_mode == "random_horizontal_flip":
+                image_transforms.append(transforms.RandomHorizontalFlip())
+            elif trans_mode == "random_vertical_flip":
+                image_transforms.append(transforms.RandomVerticalFlip())
+            elif trans_mode == "color_jitter":
+                if kargs is not None:
+                    image_transforms.append(transforms.ColorJitter(**kargs))
+                elif args is not None:
+                    image_transforms.append(transforms.ColorJitter(*args))
+                else:
+                    image_transforms.append(transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1))
+            elif trans_mode == "affine":
+                if kargs is not None:
+                    image_transforms.append(transforms.RandomAffine(**kargs))
+                elif args is not None:
+                    image_transforms.append(transforms.RandomAffine(*args))
+                else:
+                    image_transforms.append(
+                        transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1))
+                    )
+            elif trans_mode == "randaug":
+                if kargs is not None:
+                    image_transforms.append(RandAugment(**kargs))
+                elif args is not None:
+                    image_transforms.append(RandAugment(*args))
+                else:
+                    image_transforms.append(RandAugment(2, 9))
+            elif trans_mode == "trivial_augment":
+                image_transforms.append(TrivialAugment(IMAGE, 30))
+            else:
+                raise ValueError(f"unknown transform type: {trans_mode}")
+
+        return image_transforms
+
+    def construct_image_processor(
+        self,
+        image_transforms: Union[List[Callable], List[str]],
+        size: int,
+        normalization,
+    ) -> transforms.Compose:
+        """
+        Build up an image processor from the provided list of transform types.
+
+        Parameters
+        ----------
+        image_transforms
+            A list of image transform types.
+        size
+            Image size.
+        normalization
+            A transforms.Normalize object. When the image is ground truth image, 'normalization=None' should be specified.
+
+        Returns
+        -------
+        A transforms.Compose object.
+        """
+        image_transforms = self.get_image_transform_funcs(transform_types=image_transforms, size=size)
+        if not any([isinstance(trans, transforms.ToTensor) for trans in image_transforms]):
+            image_transforms.append(transforms.ToTensor())
+        if (
+            not any([isinstance(trans, transforms.Normalize) for trans in image_transforms])
+            and normalization is not None
+        ):
+            image_transforms.append(normalization)
+        return transforms.Compose(image_transforms)
+
     def __call__(
         self,
         images: Dict[str, List[str]],
-        feature_modalities: Dict[str, Union[int, float, list]],
+        sub_dtypes: Dict[str, str],
         is_training: bool,
     ) -> Dict:
         """
@@ -321,8 +361,8 @@ class ImageProcessor:
         ----------
         images
             Images of one sample.
-        feature_modalities
-            The modality of the feature columns.
+        sub_dtypes
+            The sub data types of all image columns.
         is_training
             Whether to process images in the training mode.
 
@@ -332,7 +372,7 @@ class ImageProcessor:
         """
         images = {k: [v] if isinstance(v, str) else v for k, v in images.items()}
 
-        return self.process_one_sample(images, feature_modalities, is_training)
+        return self.process_one_sample(images=images, sub_dtypes=sub_dtypes, is_training=is_training)
 
     def __getstate__(self):
         odict = self.__dict__.copy()  # get attribute dictionary
@@ -341,12 +381,7 @@ class ImageProcessor:
 
     def __setstate__(self, state):
         self.__dict__ = state
-        if "train_transform_types" in state:  # backward compatible
-            self.train_transforms = list(self.train_transform_types)
-        if "val_transform_types" in state:
-            self.val_transforms = list(self.val_transform_types)
-
-        self.train_processor = construct_image_processor(
+        self.train_processor = self.construct_image_processor(
             image_transforms=self.train_transforms,
             size=self.size,
             normalization=self.normalization,

autogluon/multimodal/data/process_label.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any, Dict, List, Optional, Union
 
 from torch import nn
@@ -5,6 +6,8 @@ from torch import nn
 from ..constants import LABEL, MMDET_IMAGE
 from .collator import ListCollator, StackCollator
 
+logger = logging.getLogger(__name__)
+
 
 class LabelProcessor:
     """
@@ -23,6 +26,7 @@ class LabelProcessor:
         model
             The model for which this processor would be created.
         """
+        logger.debug(f"initializing label processor for model {model.prefix}")
         self.prefix = model.prefix
 
     @property
@@ -68,7 +72,7 @@ class LabelProcessor:
     def __call__(
         self,
         labels: Dict[str, Union[int, float]],
-        feature_modalities: Dict[str, Union[int, float, list]],
+        sub_dtypes: Dict[str, str],
         is_training: bool,
         load_only: bool = False,  # TODO: refactor mmdet_image and remove this
     ) -> Dict:
@@ -79,8 +83,8 @@ class LabelProcessor:
         ----------
         labels
             Labels of one sample.
-        feature_modalities
-            The modality of the feature columns.
+        sub_dtypes
+            The sub data types of all label columns.
         is_training
             Whether to do processing in the training mode. This unused flag is for the API compatibility.
         load_only

autogluon/multimodal/data/process_mmlab/process_mmdet.py

@@ -6,14 +6,7 @@ import PIL
 from PIL import ImageFile
 from torch import nn
 
-try:
-    from torchvision.transforms import InterpolationMode
-
-    BICUBIC = InterpolationMode.BICUBIC
-except ImportError:
-    BICUBIC = PIL.Image.BICUBIC
-
-from ..utils import is_rois_input
+from ..infer_types import is_rois_input
 from .process_mmlab_base import MMLabProcessor
 
 try:

autogluon/multimodal/data/process_mmlab/process_mmlab_base.py

@@ -7,16 +7,9 @@ import PIL
 from PIL import ImageFile
 from torch import nn
 
-try:
-    from torchvision.transforms import InterpolationMode
-
-    BICUBIC = InterpolationMode.BICUBIC
-except ImportError:
-    BICUBIC = PIL.Image.BICUBIC
-
-from ...constants import AUTOMM, COLUMN, IMAGE, IMAGE_VALID_NUM, MMDET_IMAGE
+from ...constants import COLUMN, IMAGE, IMAGE_VALID_NUM, MMDET_IMAGE
 from ..collator import StackCollator
-from ..utils import is_rois_input
+from ..infer_types import is_rois_input
 
 try:
     with warnings.catch_warnings():

autogluon/multimodal/data/process_mmlab/process_mmocr.py

@@ -6,15 +6,7 @@ import PIL
 from PIL import ImageFile
 from torch import nn
 
-try:
-    from torchvision.transforms import InterpolationMode
-
-    BICUBIC = InterpolationMode.BICUBIC
-except ImportError:
-    BICUBIC = PIL.Image.BICUBIC
-
-from ...constants import AUTOMM
-from ..utils import is_rois_input
+from ..infer_types import is_rois_input
 from .process_mmlab_base import MMLabProcessor
 
 logger = logging.getLogger(__name__)