autogluon.multimodal 1.2.1b20250303__py3-none-any.whl → 1.2.1b20250304__py3-none-any.whl

autogluon/multimodal/data/process_ner.py

@@ -1,16 +1,18 @@
 import logging
 import os
+import re
 import warnings
 from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
 from numpy.typing import NDArray
 from omegaconf import DictConfig
+from tokenizers import pre_tokenizers
 from torch import nn
 
-from ..constants import AUTOMM, NER_ANNOTATION, NER_TEXT, TEXT, TEXT_NER
+from ..constants import NER_ANNOTATION, NER_TEXT, TEXT, TEXT_NER
+from ..models.utils import get_pretrained_tokenizer
 from .collator import PadCollator, StackCollator
-from .utils import process_ner_annotations, tokenize_ner_text
 
 logger = logging.getLogger(__name__)
 
@@ -124,12 +126,12 @@ class NerProcessor:
         ner_text = all_features[text_column]
         if is_training or annotation_column is not None:
             ner_annotation = all_features[annotation_column]
-            label, col_tokens, token_to_word_mappings, word_offsets = process_ner_annotations(
+            label, col_tokens, token_to_word_mappings, word_offsets = self.process_ner_annotations(
                 ner_annotation, ner_text, self.entity_map, self.tokenizer
             )
             ret.update({self.label_key: label})
         else:
-            col_tokens, token_to_word_mappings, word_offsets = tokenize_ner_text(ner_text, self.tokenizer)
+            col_tokens, token_to_word_mappings, word_offsets = self.tokenize_ner_text(ner_text, self.tokenizer)
             ret.update({self.label_key: np.array([], dtype=np.int32)})
 
         ret.update(
@@ -144,6 +146,192 @@ class NerProcessor:
 
         return ret
 
+    @classmethod
+    def process_ner_annotations(cls, ner_annotations, ner_text, entity_map, tokenizer, is_eval=False):
+        """
+        Generate token-level/word-level labels with given text and NER annotations.
+
+        Parameters
+        ----------
+        ner_annotations
+            The NER annotations.
+        ner_text
+            The corresponding raw text.
+        entity_map
+            The map between tags and tag indexes. e.g., {"PER":2, "LOC":3}.
+        tokenizer
+            The tokenizer to be used.
+        is_eval
+            Whether it is for evaluation or not, default: False
+
+        Returns
+        -------
+        Token-level/word-level labels and text features.
+        """
+        col_tokens, token_to_word_mappings, word_offsets = cls.tokenize_ner_text(ner_text, tokenizer)
+        num_words = len(set(token_to_word_mappings)) - 1
+        word_label = [1] * num_words
+        # TODO: Potentially optimize word label generation via binary search
+        b_prefix = "B-"
+        i_prefix = "I-"
+        for annot in ner_annotations:
+            custom_offset = annot[0]
+            custom_label = annot[1]
+            is_start_word = True
+            for idx, word_offset in enumerate(word_offsets[:num_words, :]):
+                # support multiple words in an annotated offset range.
+                # Allow partial overlapping between custom annotations and pretokenized words.
+                if (word_offset[0] < custom_offset[1]) and (custom_offset[0] < word_offset[1]):
+                    if not (
+                        re.match(b_prefix, custom_label, re.IGNORECASE)
+                        or re.match(i_prefix, custom_label, re.IGNORECASE)
+                    ):
+                        if is_start_word and b_prefix + custom_label in entity_map:
+                            word_label[idx] = entity_map[b_prefix + custom_label]
+                            is_start_word = False
+                        elif i_prefix + custom_label in entity_map:
+                            word_label[idx] = entity_map[i_prefix + custom_label]
+                    else:
+                        if custom_label in entity_map:
+                            word_label[idx] = entity_map[custom_label]
+
+        token_label = [0] * len(col_tokens.input_ids)
+        temp = set()
+        counter = 0
+        for idx, token_to_word in enumerate(token_to_word_mappings):
+            if token_to_word != -1 and token_to_word not in temp:
+                temp.add(token_to_word)
+                token_label[idx] = word_label[counter]
+                counter += 1
+        if not is_eval:
+            label = token_label  # return token-level labels for training
+        else:
+            label = word_label  # return word-level labels for evaluation
+
+        return label, col_tokens, token_to_word_mappings, word_offsets
+
+    @classmethod
+    def tokenize_ner_text(cls, text, tokenizer):
+        """
+        Tokenization process for the NER task. It will be used for the token-level label generation
+        and the input text tokenization.
+
+        Parameters
+        ----------
+        text
+            The raw text data.
+        tokenizer
+            The tokenizer to be used.
+
+        Returns
+        -------
+        The output of tokenizer and word offsets.
+        """
+        # pre-tokenization is required for NER token-level label generation.
+        words_with_offsets = pre_tokenizers.BertPreTokenizer().pre_tokenize_str(text)
+        words_with_offsets = (
+            cls.is_space_counted(words_with_offsets) if len(words_with_offsets) > 1 else words_with_offsets
+        )
+        words = [word for word, offset in words_with_offsets]
+        word_offsets = np.array([[offset[0], offset[1]] for word, offset in words_with_offsets], dtype=np.int32)
+        col_tokens = tokenizer(
+            words,
+            is_split_into_words=True,
+            return_offsets_mapping=True,
+            padding="max_length",
+            truncation=True,
+            max_length=tokenizer.model_max_length,
+            return_token_type_ids=True,
+        )
+        offset_mapping = np.array(col_tokens.offset_mapping, dtype=np.int32)
+        if len(words_with_offsets) > 1:
+            if offset_mapping.shape[0] > len(words):
+                word_offsets = np.pad(word_offsets, ((0, offset_mapping.shape[0] - len(words)), (0, 0)), "constant")
+            # token to word mappings: it will tell us which token belongs to which word.
+            token_to_word_mappings = [i if i != None else -1 for i in col_tokens.word_ids()]
+            if len(set(token_to_word_mappings)) != len(words) + 1:
+                warnings.warn(f"The token to word mappings are incorrect!")
+        else:
+            # If pre_tokenizer does not give word offsets, use word_ids and offset_mappings instead.
+            word_offsets = np.append(offset_mapping[1:], [[0, 0]], axis=0)
+            word_idx = np.arange(len(col_tokens.word_ids()) - col_tokens.word_ids().count(None))
+            token_to_word_mappings = [
+                val + word_idx[idx - 1] if val != None else -1 for idx, val in enumerate(col_tokens.word_ids())
+            ]
+
+        return col_tokens, token_to_word_mappings, word_offsets
+
+    @staticmethod
+    def is_space_counted(words_with_offsets):
+        """
+        Some tokenizers will count space into words for example.
+        Given text: 'hello world', normal bert will output: [('hello', (0, 5)), ('world', (6, 11))]
+        while some checkpoint will output: [('▁hello', (0, 5)), ('▁world', (5, 11))]
+        This will lead to inconsistency issue during labelling, details can be found here:
+        https://github.com/huggingface/transformers/issues/18111
+
+        This function will check whether space is counted or not and realign the offset.
+        """
+        offset0, offset1 = [], []
+        for word, offset in words_with_offsets:
+            offset0.append(offset[0])
+            offset1.append(offset[1])
+
+        realign = []
+        if offset0[1:] == offset1[:-1]:  # space are counted
+            realign = [words_with_offsets[0]]
+            for word, offset in words_with_offsets[1:]:
+                if word.startswith("▁"):  # it is "Lower One Eighth Block" (U+2581) rather than lower line (U+005F).
+                    realign.append((word, (offset[0] + 1, offset[1])))
+                else:
+                    realign.append((word, offset))
+
+        if realign:
+            return realign
+        else:
+            return words_with_offsets
+
+    def save_tokenizer(
+        self,
+        path: str,
+    ):
+        """
+        Save the text tokenizer and record its relative paths, e.g, hf_text.
+
+        Parameters
+        ----------
+        path
+            The root path of saving.
+
+        """
+        save_path = os.path.join(path, self.prefix)
+        self.tokenizer.save_pretrained(save_path)
+        self.tokenizer = self.prefix
+
+    def load_tokenizer(
+        self,
+        path: str,
+    ):
+        """
+        Load saved text tokenizers. If text/ner processors already have tokenizers,
+        then do nothing.
+
+        Parameters
+        ----------
+        path
+            The root path of loading.
+
+        Returns
+        -------
+        A list of text/ner processors with tokenizers loaded.
+        """
+        if isinstance(self.tokenizer, str):
+            load_path = os.path.join(path, self.tokenizer)
+            self.tokenizer = get_pretrained_tokenizer(
+                tokenizer_name=self.tokenizer_name,
+                checkpoint_name=load_path,
+            )
+
     def __call__(
         self,
         all_features: Dict[str, Union[NDArray, list]],

autogluon/multimodal/data/process_numerical.py

@@ -1,3 +1,5 @@
+import logging
+import random
 from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
@@ -6,6 +8,8 @@ from torch import nn
 from ..constants import COLUMN, NUMERICAL
 from .collator import StackCollator
 
+logger = logging.getLogger(__name__)
+
 
 class NumericalProcessor:
     """
@@ -19,6 +23,7 @@ class NumericalProcessor:
         model: nn.Module,
         merge: Optional[str] = "concat",
         requires_column_info: bool = False,
+        dropout: Optional[float] = 0,
     ):
         """
         Parameters
@@ -33,9 +38,16 @@ class NumericalProcessor:
         requires_column_info
             Whether to require feature column information in dataloader.
         """
+        logger.debug(f"initializing numerical processor for model {model.prefix}")
         self.prefix = model.prefix
         self.merge = merge
         self.requires_column_info = requires_column_info
+        self.numerical_fill_values = model.numerical_fill_values
+        self.dropout = dropout
+        assert 0 <= self.dropout <= 1
+        if self.dropout > 0:
+            logger.debug(f"numerical value dropout probability: {self.dropout}")
+            logger.debug(f"dropped values will be replaced by {self.numerical_fill_values}")
 
     @property
     def numerical_key(self):
@@ -67,6 +79,7 @@ class NumericalProcessor:
     def process_one_sample(
         self,
         numerical_features: Dict[str, float],
+        is_training: bool,
     ) -> Dict:
         """
         Process one sample's numerical features.
@@ -76,6 +89,8 @@ class NumericalProcessor:
         ----------
         numerical_features
             Numerical features of one sample.
+        is_training
+            Whether to do processing in the training mode.
 
         Returns
         -------
@@ -87,6 +102,15 @@ class NumericalProcessor:
             for i, col_name in enumerate(numerical_features.keys()):
                 ret[f"{self.numerical_column_prefix}_{col_name}"] = i
 
+        if is_training and self.dropout > 0:
+            numerical_features_copy = dict()
+            for k, v in numerical_features.items():
+                if random.uniform(0, 1) <= self.dropout:
+                    numerical_features_copy[k] = self.numerical_fill_values[k]
+                else:
+                    numerical_features_copy[k] = v
+            numerical_features = numerical_features_copy
+
         if self.merge == "concat":
             ret[self.numerical_key] = np.array(list(numerical_features.values()), dtype=np.float32)
         else:
@@ -97,7 +121,7 @@ class NumericalProcessor:
     def __call__(
         self,
         numerical_features: Dict[str, float],
-        feature_modalities: Dict[str, Union[int, float, list]],
+        sub_dtypes: Dict[str, str],
         is_training: bool,
     ) -> Dict:
         """
@@ -107,13 +131,16 @@ class NumericalProcessor:
         ----------
         numerical_features
             Numerical features of one sample.
-        feature_modalities
-            The modality of the feature columns.
+        sub_dtypes
+            The sub data types of all numerical columns.
         is_training
-            Whether to do processing in the training mode. This unused flag is for the API compatibility.
+            Whether to do processing in the training mode.
 
         Returns
         -------
         A dictionary containing one sample's processed numerical features.
         """
-        return self.process_one_sample(numerical_features)
+        return self.process_one_sample(
+            numerical_features=numerical_features,
+            is_training=is_training,
+        )

autogluon/multimodal/data/process_semantic_seg_img.py

@@ -1,44 +1,32 @@
 import logging
 import random
-import warnings
-from io import BytesIO
 from typing import Dict, List, Optional, Union
 
 import numpy as np
 import PIL
 import torch
-from omegaconf import DictConfig
 from PIL import Image, ImageFile
 from torch import nn
 from torchvision import transforms
 
-from .utils import construct_image_processor, image_mean_std
-
-try:
-    from torchvision.transforms import InterpolationMode
-
-    BICUBIC = InterpolationMode.BICUBIC
-except ImportError:
-    BICUBIC = PIL.Image.BICUBIC
-
 from ..constants import (
     CLASS_LABEL,
     COLUMN,
     IMAGE,
-    IMAGE_BYTEARRAY,
     IMAGE_VALID_NUM,
     LABEL,
     MASK_LABEL,
     SEMANTIC_SEGMENTATION_GT,
     SEMANTIC_SEGMENTATION_IMG,
 )
-from .collator import ListCollator, PadCollator, StackCollator
+from .collator import ListCollator, PadCollator
+from .process_image import ImageProcessor
 
 logger = logging.getLogger(__name__)
 ImageFile.LOAD_TRUNCATED_IMAGES = True
 
 
-class SemanticSegImageProcessor:
+class SemanticSegImageProcessor(ImageProcessor):
     """
     Prepare image data for the model specified by "prefix". For multiple models requiring image data,
     we need to create a ImageProcessor for each related model so that they will have independent input.
@@ -51,7 +39,6 @@ class SemanticSegImageProcessor:
         gt_transforms: List[str],
         train_transforms: Optional[List[str]] = None,
         val_transforms: Optional[List[str]] = None,
-        norm_type: Optional[str] = None,
         max_img_num_per_col: Optional[int] = 1,
         missing_value_strategy: Optional[str] = "skip",
         requires_column_info: bool = False,
@@ -70,15 +57,6 @@ class SemanticSegImageProcessor:
             A list of image transforms used in training for data augmentation. Note that the transform order matters.
         val_transforms
             A list of image transforms used in validation/test/prediction. Note that the transform order matters.
-        norm_type
-            How to normalize an image. We now support:
-            - inception
-                Normalize image by IMAGENET_INCEPTION_MEAN and IMAGENET_INCEPTION_STD from timm
-            - imagenet
-                Normalize image by IMAGENET_DEFAULT_MEAN and IMAGENET_DEFAULT_STD from timm
-            - clip
-                Normalize image by mean (0.48145466, 0.4578275, 0.40821073) and
-                std (0.26862954, 0.26130258, 0.27577711), used for CLIP.
         max_img_num_per_col
             The maximum number of images one sample can have.
         missing_value_strategy
@@ -98,7 +76,8 @@ class SemanticSegImageProcessor:
         self.requires_column_info = requires_column_info
 
         self.size = model.image_size
-        self.mean, self.std = image_mean_std(norm_type)
+        self.mean = model.image_mean
+        self.std = model.image_std
         self.normalization = transforms.Normalize(self.mean, self.std)
         self.num_classes = model.num_classes
         self.ignore_label = ignore_label
@@ -110,10 +89,10 @@ class SemanticSegImageProcessor:
         self.max_img_num_per_col = max_img_num_per_col
         logger.debug(f"max_img_num_per_col: {max_img_num_per_col}")
 
-        self.img_processor = construct_image_processor(
+        self.img_processor = self.construct_image_processor(
             image_transforms=self.img_transforms, size=self.size, normalization=self.normalization
         )
-        self.gt_processor = construct_image_processor(
+        self.gt_processor = self.construct_image_processor(
             image_transforms=self.gt_transforms, size=self.size, normalization=None
         )
         self.train_transforms = self.get_train_transforms(train_transforms)
@@ -325,3 +304,19 @@ class SemanticSegImageProcessor:
             if trans_mode == "random_horizontal_flip":
                 train_trans.append(transforms.RandomHorizontalFlip(1.0))
         return transforms.Compose(train_trans)
+
+    def __getstate__(self):
+        odict = self.__dict__.copy()  # get attribute dictionary
+        del odict["img_processor"]
+        del odict["gt_processor"]
+
+        return odict
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+        self.img_processor = self.construct_image_processor(
+            image_transforms=self.img_transforms, size=self.size, normalization=self.normalization
+        )
+        self.gt_processor = self.construct_image_processor(
+            image_transforms=self.gt_transforms, size=self.size, normalization=None
+        )