autogluon.multimodal 1.2.1b20250303__py3-none-any.whl → 1.2.1b20250305__py3-none-any.whl

autogluon/multimodal/data/preprocess_dataframe.py

@@ -14,17 +14,14 @@ from sklearn.preprocessing import MinMaxScaler, StandardScaler
 from autogluon.features import CategoryFeatureGenerator
 
 from ..constants import (
-    AUTOMM,
     CATEGORICAL,
     DOCUMENT,
-    DOCUMENT_IMAGE,
     IDENTIFIER,
     IMAGE,
     IMAGE_BASE64_STR,
     IMAGE_BYTEARRAY,
     IMAGE_PATH,
     LABEL,
-    NER,
     NER_ANNOTATION,
     NULL,
     NUMERICAL,
@@ -73,19 +70,17 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
 
         if label_column:
             if label_generator is None:
-                self._label_generator = CustomLabelEncoder(
-                    positive_class=OmegaConf.select(config, "pos_label", default=None)
-                )
+                self._label_generator = CustomLabelEncoder(positive_class=config.pos_label)
             else:
                 self._label_generator = label_generator
 
             # Scaler used for numerical labels
-            numerical_label_preprocessing = OmegaConf.select(config, "label.numerical_label_preprocessing")
+            numerical_label_preprocessing = config.label.numerical_preprocessing
             if numerical_label_preprocessing == "minmaxscaler":
                 self._label_scaler = MinMaxScaler()
             elif numerical_label_preprocessing == "standardscaler":
                 self._label_scaler = StandardScaler()
-            elif numerical_label_preprocessing is None or numerical_label_preprocessing.lower() == "none":
+            elif numerical_label_preprocessing is None:
                 self._label_scaler = StandardScaler(with_mean=False, with_std=False)
             else:
                 raise ValueError(
@@ -135,8 +130,7 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
         # Some columns will be ignored
         self._ignore_columns_set = set()
         self._text_feature_names = []
-        self._categorical_feature_names = []
-        self._categorical_num_categories = []
+        self._categorical_num_categories = dict()
         self._numerical_feature_names = []
         self._image_feature_names = []
         self._rois_feature_names = []
@@ -154,10 +148,7 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
 
     @property
     def image_path_names(self):
-        if hasattr(self, "_image_path_names"):
-            return self._image_path_names
-        else:
-            return [col_name for col_name in self._image_feature_names if self._column_types[col_name] == IMAGE_PATH]
+        return [col_name for col_name in self._image_feature_names if self._column_types[col_name] == IMAGE_PATH]
 
     @property
     def rois_feature_names(self):
@@ -173,7 +164,7 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
 
     @property
     def image_feature_names(self):
-        return self._image_path_names if hasattr(self, "_image_path_names") else self._image_feature_names
+        return self._image_feature_names
 
     @property
     def text_feature_names(self):
@@ -181,12 +172,21 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
 
     @property
     def categorical_feature_names(self):
-        return self._categorical_feature_names
+        return list(self.categorical_num_categories.keys())
 
     @property
     def numerical_feature_names(self):
         return self._numerical_feature_names
 
+    @property
+    def numerical_fill_values(self):
+        ret = dict()
+        for col_name in self._numerical_feature_names:
+            generator = self._feature_generators[col_name]
+            ret[col_name] = generator.transform(np.full([1, 1], np.nan))[:, 0][0]
+
+        return ret
+
     @property
     def document_feature_names(self):
         # Added for backward compatibility.
@@ -216,17 +216,12 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
 
     @property
     def required_feature_names(self):
-        image_feature_names = (
-            self._image_path_names if hasattr(self, "_image_path_names") else self._image_feature_names
-        )
-        rois_feature_names = self._rois_feature_names if hasattr(self, "_rois_feature_names") else []
-
         return (
-            image_feature_names
+            self._image_feature_names
             + self._text_feature_names
             + self._numerical_feature_names
-            + self._categorical_feature_names
-            + rois_feature_names
+            + self.categorical_feature_names
+            + self._rois_feature_names
         )
 
     @property
@@ -268,16 +263,13 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
 
     def get_column_names(self, modality: str):
         if modality.startswith(IMAGE):
-            if hasattr(self, "_image_path_names"):
-                return self._image_path_names
-            else:
-                return self._image_feature_names
+            return self._image_feature_names
         elif modality == ROIS:
             return self._rois_feature_names
         elif modality == TEXT:
             return self._text_feature_names
         elif modality == CATEGORICAL:
-            return self._categorical_feature_names
+            return self.categorical_feature_names
         elif modality == NUMERICAL:
             return self._numerical_feature_names
         elif modality.startswith(DOCUMENT):
@@ -344,8 +336,7 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
                         continue
                     num_categories = len(generator.category_map[col_name])
                     # Add one unknown category
-                    self._categorical_num_categories.append(num_categories + 1)
-                    self._categorical_feature_names.append(col_name)
+                    self._categorical_num_categories[col_name] = num_categories + 1
             elif col_type == NUMERICAL:
                 processed_data = pd.to_numeric(col_value)
                 if len(processed_data.unique()) == 1:
@@ -392,7 +383,7 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
         elif self.label_type == NUMERICAL:
             y = pd.to_numeric(y).to_numpy()
             self._label_scaler.fit(np.expand_dims(y, axis=-1))
-        elif self.label_type == ROIS or self.label_type == SEMANTIC_SEGMENTATION_GT:
+        elif self.label_type in [ROIS, SEMANTIC_SEGMENTATION_GT]:
             pass  # Do nothing. TODO: Shall we call fit here?
         elif self.label_type == NER_ANNOTATION:
             # If there are ner annotations and text columns but no NER feature columns,
@@ -426,6 +417,24 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
         if y is not None:
             self._fit_y(y=y, X=X)
 
+    @staticmethod
+    def convert_categorical_to_text(col_value: pd.Series, template: str, col_name: str):
+        # TODO: do we need to consider whether categorical values are valid text?
+        col_value = col_value.astype("object")
+        if template == "direct":
+            processed_data = col_value.apply(lambda ele: "" if pd.isnull(ele) else str(ele))
+        elif template == "list":
+            processed_data = col_value.apply(lambda ele: "" if pd.isnull(ele) else col_name + ": " + str(ele))
+        elif template == "text":
+            processed_data = col_value.apply(lambda ele: "" if pd.isnull(ele) else col_name + " is " + str(ele))
+        elif template == "latex":
+            processed_data = col_value.apply(lambda ele: "" if pd.isnull(ele) else str(ele) + " & ")
+        else:
+            raise ValueError(
+                f"Unsupported template {template} for converting categorical data into text. Select one from: ['direct', 'list', 'text', 'latex']."
+            )
+        return processed_data
+
     def transform_text(
         self,
         df: pd.DataFrame,
@@ -455,10 +464,15 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
         for col_name in self._text_feature_names:
             col_value = df[col_name]
             col_type = self._column_types[col_name]
-            if col_type == TEXT or col_type == CATEGORICAL:
-                # TODO: do we need to consider whether categorical values are valid text?
+            if col_type == TEXT:
                 col_value = col_value.astype("object")
                 processed_data = col_value.apply(lambda ele: "" if pd.isnull(ele) else str(ele))
+            elif col_type == CATEGORICAL:
+                processed_data = self.convert_categorical_to_text(
+                    col_value=col_value,
+                    template=self._config.categorical.convert_to_text_template,
+                    col_name=col_name,
+                )
             elif col_type == NUMERICAL:
                 processed_data = pd.to_numeric(col_value).apply("{:.3f}".format)
             elif col_type == f"{TEXT}_{IDENTIFIER}":
@@ -710,7 +724,7 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
             self._fit_called or self._fit_x_called
         ), "You will need to first call preprocessor.fit before calling preprocessor.transform_categorical."
         categorical_features = {}
-        for col_name, num_category in zip(self._categorical_feature_names, self._categorical_num_categories):
+        for col_name, num_category in self._categorical_num_categories.items():
             col_value = df[col_name]
             processed_data = col_value.astype("category")
             generator = self._feature_generators[col_name]
@@ -757,7 +771,7 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
         elif self.label_type == NUMERICAL:
             y = pd.to_numeric(y_df).to_numpy()
             y = self._label_scaler.transform(np.expand_dims(y, axis=-1))[:, 0].astype(np.float32)
-        elif self.label_type == ROIS or self.label_type == SEMANTIC_SEGMENTATION_GT:
+        elif self.label_type in [ROIS, SEMANTIC_SEGMENTATION_GT]:
             y = y_df.to_list()
         elif self.label_type == NER_ANNOTATION:
             y = self._label_generator.transform(y_df)
@@ -866,8 +880,11 @@ class MultiModalFeaturePreprocessor(TransformerMixin, BaseEstimator):
         ), "You will need to first call preprocessor.fit_y() before calling preprocessor.transform_prediction."
 
         if self.label_type == CATEGORICAL:
-            assert y_pred.shape[1] >= 2
-            y_pred = y_pred.argmax(axis=1)
+            assert len(y_pred.shape) <= 2
+            if len(y_pred.shape) == 2 and y_pred.shape[1] >= 2:
+                y_pred = y_pred.argmax(axis=1)
+            else:
+                y_pred = (y_pred > 0.5).astype(int)
             # Transform the predicted label back to the original space (e.g., string values)
             if inverse_categorical:
                 y_pred = self._label_generator.inverse_transform(y_pred)

autogluon/multimodal/data/process_categorical.py

@@ -1,11 +1,14 @@
+import logging
+import random
 from typing import Any, Dict, List, Optional, Union
 
-import numpy as np
 from torch import nn
 
 from ..constants import CATEGORICAL, COLUMN
 from .collator import StackCollator, TupleCollator
 
+logger = logging.getLogger(__name__)
+
 
 class CategoricalProcessor:
     """
@@ -18,6 +21,7 @@ class CategoricalProcessor:
         self,
         model: nn.Module,
         requires_column_info: bool = False,
+        dropout: Optional[float] = 0,
     ):
         """
         Parameters
@@ -27,8 +31,16 @@ class CategoricalProcessor:
         requires_column_info
             Whether to require feature column information in dataloader.
         """
+        logger.debug(f"initializing categorical processor for model {model.prefix}")
         self.prefix = model.prefix
         self.requires_column_info = requires_column_info
+        self.num_categories = model.num_categories
+        self.dropout = dropout
+        assert 0 <= self.dropout <= 1
+        if self.dropout > 0:
+            logger.debug(f"categorical value dropout probability: {self.dropout}")
+            fill_values = {k: v - 1 for k, v in self.num_categories.items()}
+            logger.debug(f"dropped values will be replaced by {fill_values}")
 
     @property
     def categorical_key(self):
@@ -60,6 +72,7 @@ class CategoricalProcessor:
     def process_one_sample(
         self,
         categorical_features: Dict[str, int],
+        is_training: bool,
     ) -> Dict:
         """
         Process one sample's categorical features. Assume the categorical features
@@ -69,6 +82,8 @@ class CategoricalProcessor:
         ----------
         categorical_features
             Categorical features of one sample.
+        is_training
+            Whether to do processing in the training mode.
 
         Returns
         -------
@@ -80,6 +95,17 @@ class CategoricalProcessor:
             for i, col_name in enumerate(categorical_features.keys()):
                 ret[f"{self.categorical_column_prefix}_{col_name}"] = i
 
+        if is_training and self.dropout > 0:
+            categorical_features_copy = dict()
+            for k, v in categorical_features.items():
+                if random.uniform(0, 1) <= self.dropout:
+                    categorical_features_copy[k] = self.num_categories[k] - 1
+                else:
+                    categorical_features_copy[k] = v
+            categorical_features = categorical_features_copy
+
+        # make sure keys are in the same order
+        assert list(categorical_features.keys()) == list(self.num_categories.keys())
         ret[self.categorical_key] = list(categorical_features.values())
 
         return ret
@@ -87,7 +113,7 @@ class CategoricalProcessor:
     def __call__(
         self,
         categorical_features: Dict[str, int],
-        feature_modalities: Dict[str, Union[int, float, list]],
+        sub_dtypes: Dict[str, str],
         is_training: bool,
     ) -> Dict:
         """
@@ -97,13 +123,16 @@ class CategoricalProcessor:
         ----------
         categorical_features
             Categorical features of one sample.
-        feature_modalities
-            The modality of the feature columns.
+        sub_dtypes
+            The sub data types of all categorical columns.
         is_training
-            Whether to do processing in the training mode. This unused flag is for the API compatibility.
+            Whether to do processing in the training mode.
 
         Returns
         -------
         A dictionary containing one sample's processed categorical features.
         """
-        return self.process_one_sample(categorical_features)
+        return self.process_one_sample(
+            categorical_features=categorical_features,
+            is_training=is_training,
+        )

autogluon/multimodal/data/process_document.py

@@ -1,30 +1,24 @@
-import importlib.util
 import logging
 import os
-import re
-import shutil
-import subprocess
 import warnings
-from io import BytesIO
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional, Union
 
 import numpy as np
 import PIL
 import pytesseract
-import torch
 from numpy.typing import NDArray
-from PIL import ImageFile
 from torch import nn
 from torchvision import transforms
 
-from ..constants import AUTOMM, BBOX, DOCUMENT_PDF
-from .collator import PadCollator, StackCollator
-from .utils import construct_image_processor, image_mean_std
+from ..constants import BBOX, DOCUMENT_PDF
+from ..models.utils import get_pretrained_tokenizer
+from .collator import PadCollator
+from .process_image import ImageProcessor
 
 logger = logging.getLogger(__name__)
 
 
-class DocumentProcessor:
+class DocumentProcessor(ImageProcessor):
     """
     Prepare document data for Document Classification.
     OCR (Optical character recognition) is applied to get the document texts and bounding boxes.
@@ -34,9 +28,8 @@ class DocumentProcessor:
     def __init__(
         self,
         model: nn.Module,
-        train_transform_types: List[str],
-        val_transform_types: List[str],
-        norm_type: Optional[str] = None,
+        train_transforms: Union[List[str], Callable, List[Callable]],
+        val_transforms: Union[List[str], Callable, List[Callable]],
         size: Optional[int] = None,
         text_max_len: Optional[int] = 512,
         missing_value_strategy: Optional[str] = "zero",
@@ -46,19 +39,10 @@ class DocumentProcessor:
         ----------
         model
             The model using this data processor.
-        train_transform_types
+        train_transforms
             A list of image transforms used in training. Note that the transform order matters.
-        val_transform_types
+        val_transforms
             A list of image transforms used in validation/test/prediction. Note that the transform order matters.
-        norm_type
-            How to normalize an image. We now support:
-            - inception
-                Normalize image by IMAGENET_INCEPTION_MEAN and IMAGENET_INCEPTION_STD from timm
-            - imagenet
-                Normalize image by IMAGENET_DEFAULT_MEAN and IMAGENET_DEFAULT_STD from timm
-            - clip
-                Normalize image by mean (0.48145466, 0.4578275, 0.40821073) and
-                std (0.26862954, 0.26130258, 0.27577711), used for CLIP.
         size
             The width / height of a square image.
         text_max_len
@@ -79,15 +63,16 @@ class DocumentProcessor:
 
         # For document image processing.
         self.size = size
-        self.train_transform_types = train_transform_types
-        self.val_transform_types = val_transform_types
-        self.mean, self.std = image_mean_std(norm_type)
+        self.train_transforms = train_transforms
+        self.val_transforms = val_transforms
+        self.mean = model.image_mean
+        self.std = model.image_std
         self.normalization = transforms.Normalize(self.mean, self.std)
-        self.train_processor = construct_image_processor(
-            size=self.size, normalization=self.normalization, image_transforms=self.train_transform_types
+        self.train_processor = self.construct_image_processor(
+            size=self.size, normalization=self.normalization, image_transforms=self.train_transforms
         )
-        self.val_processor = construct_image_processor(
-            size=self.size, normalization=self.normalization, image_transforms=self.val_transform_types
+        self.val_processor = self.construct_image_processor(
+            size=self.size, normalization=self.normalization, image_transforms=self.val_transforms
         )
 
         self.missing_value_strategy = missing_value_strategy
@@ -359,6 +344,47 @@ class DocumentProcessor:
 
         return ret
 
+    def save_tokenizer(
+        self,
+        path: str,
+    ):
+        """
+        Save the text tokenizer and record its relative paths, e.g, hf_text.
+
+        Parameters
+        ----------
+        path
+            The root path of saving.
+
+        """
+        save_path = os.path.join(path, self.prefix)
+        self.tokenizer.save_pretrained(save_path)
+        self.tokenizer = self.prefix
+
+    def load_tokenizer(
+        self,
+        path: str,
+    ):
+        """
+        Load saved text tokenizers. If text/ner processors already have tokenizers,
+        then do nothing.
+
+        Parameters
+        ----------
+        path
+            The root path of loading.
+
+        Returns
+        -------
+        A list of text/ner processors with tokenizers loaded.
+        """
+        if isinstance(self.tokenizer, str):
+            load_path = os.path.join(path, self.tokenizer)
+            self.tokenizer = get_pretrained_tokenizer(
+                tokenizer_name=self.tokenizer_name,
+                checkpoint_name=load_path,
+            )
+
     def __call__(
         self,
         all_features: Dict[str, Union[NDArray, list]],