autogluon.multimodal 1.2.1b20250303__py3-none-any.whl → 1.2.1b20250304__py3-none-any.whl

autogluon/multimodal/utils/device.py

@@ -0,0 +1,120 @@
+import logging
+import math
+import warnings
+from typing import Dict, List, Optional, Tuple, Union
+
+import torch
+from lightning.pytorch.accelerators import find_usable_cuda_devices
+from torch import nn
+
+from autogluon.common.utils.resource_utils import ResourceManager
+
+from .env import is_interactive_env
+
+logger = logging.getLogger(__name__)
+
+
+def compute_num_gpus(config_num_gpus: Union[int, float, List], accelerator: str):
+    """
+    Compute the gpu number to initialize the lightning trainer.
+
+    Parameters
+    ----------
+    config_num_gpus
+        The gpu number provided by config.
+    accelerator
+        # "cpu", "gpu", or "auto".
+
+    Returns
+    -------
+    A valid gpu number for the current environment and config.
+    """
+    if isinstance(accelerator, str) and accelerator.lower() not in ["gpu", "auto"]:
+        return 0
+
+    config_num_gpus = (
+        math.floor(config_num_gpus) if isinstance(config_num_gpus, (int, float)) else len(config_num_gpus)
+    )
+    detected_num_gpus = ResourceManager.get_gpu_count_torch()
+
+    if config_num_gpus < 0:  # In case config_num_gpus is -1, meaning using all gpus.
+        num_gpus = detected_num_gpus
+    else:
+        num_gpus = min(config_num_gpus, detected_num_gpus)
+        if detected_num_gpus < config_num_gpus:
+            warnings.warn(
+                f"Using the detected GPU number {detected_num_gpus}, "
+                f"smaller than the GPU number {config_num_gpus} in the config.",
+                UserWarning,
+            )
+
+    return num_gpus
+
+
+def move_to_device(obj: Union[torch.Tensor, nn.Module, Dict, List, Tuple], device: torch.device):
+    """
+    Move an object to the given device.
+
+    Parameters
+    ----------
+    obj
+        An object, which can be a tensor, a module, a dict, or a list.
+    device
+        A Pytorch device instance.
+
+    Returns
+    -------
+    The object on the device.
+    """
+    if not isinstance(device, torch.device):
+        raise ValueError(f"Invalid device: {device}. Ensure the device type is `torch.device`.")
+
+    if torch.is_tensor(obj) or isinstance(obj, nn.Module):
+        return obj.to(device)
+    elif isinstance(obj, dict):
+        res = {}
+        for k, v in obj.items():
+            res[k] = move_to_device(v, device)
+        return res
+    elif isinstance(obj, list) or isinstance(obj, tuple):
+        res = []
+        for v in obj:
+            res.append(move_to_device(v, device))
+        return res
+    elif isinstance(obj, (int, float, str)):
+        return obj
+    else:
+        raise TypeError(
+            f"Invalid type {type(obj)} for move_to_device. "
+            f"Make sure the object is one of these: a Pytorch tensor, a Pytorch module, "
+            f"a dict or list of tensors or modules."
+        )
+
+
+def get_available_devices(num_gpus: int, auto_select_gpus: bool):
+    """
+    Get the available devices.
+
+    Parameters
+    ----------
+    num_gpus
+        Number of GPUs.
+    auto_select_gpus
+        Whether to pick GPU indices that are "accessible". See here: https://github.com/Lightning-AI/lightning/blob/accd2b9e61063ba3c683764043030545ed87c71f/src/lightning/fabric/accelerators/cuda.py#L79
+
+    Returns
+    -------
+    The available devices.
+    """
+    if num_gpus > 0:
+        if auto_select_gpus:
+            if is_interactive_env():
+                devices = list(range(num_gpus))
+            else:
+                devices = find_usable_cuda_devices(num_gpus)
+        else:
+            devices = num_gpus
+    else:
+        devices = "auto"
+
+    return devices

autogluon/multimodal/utils/distillation.py

@@ -5,9 +5,9 @@ from omegaconf import DictConfig, OmegaConf
 from torch import nn
 
 from ..constants import REGRESSION
-from ..optimization.losses import RKDLoss
-from ..utils.data import turn_on_off_feature_column_info
-from ..utils.model import modify_duplicate_model_names
+from ..data import turn_on_off_feature_column_info
+from ..models import modify_duplicate_model_names
+from ..optim.losses import RKDLoss
 
 logger = logging.getLogger(__name__)
 
@@ -79,7 +79,7 @@ class DistillationMixin:
         else:
             raise ValueError(f"Unknown soft_label_loss_type: {config.distiller.softmax_regression_loss_type}")
 
-        output_feature_loss_type = OmegaConf.select(config, "distiller.output_feature_loss_type", default="mse")
+        output_feature_loss_type = config.distiller.output_feature_loss_type
         if output_feature_loss_type == "cosine":
             output_feature_loss_func = nn.CosineEmbeddingLoss()
         elif output_feature_loss_type == "mse":
@@ -97,11 +97,11 @@ class DistillationMixin:
             else nn.Identity()
         )
 
-        rkd_distance_loss_weight = OmegaConf.select(config, "distiller.rkd_distance_loss_weight", default=0.0)
-        rkd_angle_loss_weight = OmegaConf.select(config, "distiller.rkd_angle_loss_weight", default=0.0)
+        rkd_distance_loss_weight = config.distiller.rkd_distance_loss_weight
+        rkd_angle_loss_weight = config.distiller.rkd_angle_loss_weight
         rkd_loss_func = RKDLoss(rkd_distance_loss_weight, rkd_angle_loss_weight)
-        output_feature_loss_weight = OmegaConf.select(config, "distiller.output_feature_loss_weight", default=0.0)
-        softmax_regression_weight = OmegaConf.select(config, "distiller.softmax_regression_weight", default=0.0)
+        output_feature_loss_weight = config.distiller.output_feature_loss_weight
+        softmax_regression_weight = config.distiller.softmax_regression_weight
 
         # turn on returning column information in data processors
         turn_on_off_feature_column_info(

autogluon/multimodal/utils/download.py

@@ -12,7 +12,7 @@ import boto3
 import requests
 import tqdm
 
-from ..constants import AUTOMM, S3_PREFIX
+from ..constants import S3_PREFIX
 
 logger = logging.getLogger(__name__)
 

autogluon/multimodal/utils/env.py

@@ -0,0 +1,22 @@
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Union
+
+from fsspec.core import url_to_fs
+from fsspec.implementations.local import AbstractFileSystem
+
+logger = logging.getLogger(__name__)
+
+
+def is_interactive_env():
+    """
+    Return whether the current process is running under the interactive mode.
+    Check also https://stackoverflow.com/a/64523765
+    """
+    return hasattr(sys, "ps1")
+
+
+def get_filesystem(path: Union[str, Path], **kwargs: Any) -> AbstractFileSystem:
+    fs, _ = url_to_fs(str(path), **kwargs)
+    return fs

autogluon/multimodal/utils/export.py

@@ -10,11 +10,11 @@ import torch
 
 from ..constants import CATEGORICAL, HF_TEXT, IMAGE_PATH, MMDET_IMAGE, NULL, NUMERICAL, TEXT, TIMM_IMAGE
 from ..models.fusion import AbstractMultimodalFusionModel
-from ..models.huggingface_text import HFAutoModelForTextPrediction
+from ..models.hf_text import HFAutoModelForTextPrediction
 from ..models.mmdet_image import MMDetAutoModelForObjectDetection
 from ..models.timm_image import TimmAutoModelForImagePrediction
-from .environment import infer_precision
 from .onnx import OnnxModule, onnx_get_dynamic_axes
+from .precision import infer_precision
 
 logger = logging.getLogger(__name__)
 
@@ -110,7 +110,7 @@ class ExportMixin:
         import torch.jit
 
         from ..models.fusion.fusion_mlp import MultimodalFusionMLP
-        from ..models.huggingface_text import HFAutoModelForTextPrediction
+        from ..models.hf_text import HFAutoModelForTextPrediction
         from ..models.timm_image import TimmAutoModelForImagePrediction
 
         supported_models = (TimmAutoModelForImagePrediction, HFAutoModelForTextPrediction, MultimodalFusionMLP)

autogluon/multimodal/utils/hpo.py

@@ -8,8 +8,8 @@ import yaml
 from autogluon.common.utils.context import set_torch_num_threads
 
 from ..constants import BEST_K_MODELS_FILE, RAY_TUNE_CHECKPOINT
+from ..models import create_fusion_model
 from .matcher import create_siamese_model
-from .model import create_fusion_model
 
 logger = logging.getLogger(__name__)
 
@@ -92,10 +92,10 @@ def build_final_learner(
     The constructed learner.
     """
     if is_matching:
-        from ..learners.matching import MultiModalMatcher
+        from ..learners import MatchingLearner
 
         # reload the learner metadata
-        matcher = MultiModalMatcher._load_metadata(matcher=learner, path=best_trial_path)
+        matcher = MatchingLearner._load_metadata(matcher=learner, path=best_trial_path)
         # construct the model
         matcher._query_model, matcher._response_model = create_siamese_model(
             query_config=matcher._query_config,
@@ -106,7 +106,7 @@ def build_final_learner(
         matcher.top_k_average(
             save_path=best_trial_path,
             last_ckpt_path=last_ckpt_path,
-            top_k_average_method=matcher._config.optimization.top_k_average_method,
+            top_k_average_method=matcher._config.optim.top_k_average_method,
         )
         matcher._save_path = save_path
 
@@ -130,7 +130,7 @@ def build_final_learner(
         learner.top_k_average(
             save_path=best_trial_path,
             last_ckpt_path=last_ckpt_path,
-            top_k_average_method=learner._config.optimization.top_k_average_method,
+            top_k_average_method=learner._config.optim.top_k_average_method,
             standalone=standalone,
             clean_ckpts=clean_ckpts,
         )

autogluon/multimodal/utils/inference.py

@@ -3,7 +3,6 @@ from typing import Callable, Dict, List, Optional, Tuple, Union
 
 import pandas as pd
 import torch
-from omegaconf import OmegaConf
 from scipy.special import softmax
 from torch import nn
 
@@ -12,7 +11,6 @@ from ..constants import (
     COLUMN_FEATURES,
     FEATURES,
     IMAGE,
-    IMAGE_META,
     LOGITS,
     MASKS,
     NER_ANNOTATION,
@@ -29,13 +27,48 @@ from ..constants import (
 from ..data.preprocess_dataframe import MultiModalFeaturePreprocessor
 from ..data.utils import apply_data_processor, apply_df_preprocessor, get_collate_fn, get_per_sample_features
 from ..models.utils import run_model
-from .environment import get_precision_context, move_to_device
+from .device import move_to_device
 from .matcher import compute_matching_probability
 from .misc import tensor_to_ndarray
+from .precision import get_precision_context
 
 logger = logging.getLogger(__name__)
 
 
+def compute_inference_batch_size(
+    per_gpu_batch_size: int,
+    inference_batch_size_ratio: Union[int, float],
+    num_gpus: int,
+    strategy: str,
+):
+    """
+    Compute the batch size for inference.
+
+    Parameters
+    ----------
+    per_gpu_batch_size
+        Per gpu batch size from the config.
+    inference_batch_size_ratio
+        per_gpu_batch_size_for_inference = per_gpu_batch_size * inference_batch_size_ratio.
+    num_gpus
+        Number of GPUs.
+    strategy
+        A pytorch lightning strategy.
+
+    Returns
+    -------
+    Batch size for inference.
+    """
+    batch_size = per_gpu_batch_size * inference_batch_size_ratio
+
+    if num_gpus > 1 and strategy == "dp":
+        # If using 'dp', the per_gpu_batch_size would be split by all GPUs.
+        # So, we need to use the GPU number as a multiplier to compute the batch size.
+        batch_size = batch_size * num_gpus
+
+    return batch_size
+
+
 def extract_from_output(outputs: List[Dict], ret_type: str, as_ndarray: Optional[bool] = True):
     """
     Extract desired information, e.g., logits or features, from a list of model outputs.
@@ -334,7 +367,7 @@ class RealtimeMixin:
                 per_sample_features_group = apply_data_processor(
                     per_sample_features=per_sample_features_group,
                     data_processors=per_processors_group,
-                    feature_modalities=modality_types[group_id],
+                    data_types=modality_types[group_id],
                     is_training=False,
                 )
                 per_sample_features.update(per_sample_features_group)

autogluon/multimodal/utils/install.py

@@ -0,0 +1,91 @@
+import logging
+import warnings
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from ..constants import OBJECT_DETECTION, OCR
+
+logger = logging.getLogger(__name__)
+
+
+def check_if_packages_installed(problem_type: str = None, package_names: List[str] = None):
+    """
+    Check if necessary packages are installed for some problem types.
+    Raise an error if an package can't be imported.
+
+    Parameters
+    ----------
+    problem_type
+        Problem type
+    """
+    if problem_type:
+        problem_type = problem_type.lower()
+        if any(p in problem_type for p in [OBJECT_DETECTION, OCR]):
+            try:
+                with warnings.catch_warnings():
+                    warnings.simplefilter("ignore")
+                    import mmcv
+            except ImportError as e:
+                raise ValueError(
+                    f"Encountered error while importing mmcv: {e}. {_get_mmlab_installation_guide('mmcv')}"
+                )
+
+            try:
+                import mmdet
+            except ImportError as e:
+                raise ValueError(
+                    f"Encountered error while importing mmdet: {e}. {_get_mmlab_installation_guide('mmdet')}"
+                )
+
+            if OCR in problem_type:
+                try:
+                    import mmocr
+                except ImportError as e:
+                    raise ValueError(
+                        f'Encountered error while importing mmocr: {e}. Try to install mmocr: pip install "mmocr<1.0".'
+                    )
+    if package_names:
+        for package_name in package_names:
+            if package_name == "mmcv":
+                try:
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("ignore")
+                        import mmcv
+                    from mmcv import ConfigDict
+                    from mmcv.runner import load_checkpoint
+                    from mmcv.transforms import Compose
+                except ImportError as e:
+                    f"Encountered error while importing {package_name}: {e}. {_get_mmlab_installation_guide(package_name)}"
+            elif package_name == "mmdet":
+                try:
+                    import mmdet
+                    from mmdet.datasets.transforms import ImageToTensor
+                    from mmdet.registry import MODELS
+                except ImportError as e:
+                    f"Encountered error while importing {package_name}: {e}. {_get_mmlab_installation_guide(package_name)}"
+            elif package_name == "mmengine":
+                try:
+                    import mmengine
+                    from mmengine.dataset import pseudo_collate as collate
+                    from mmengine.runner import load_checkpoint
+                except ImportError as e:
+                    warnings.warn(e)
+                    raise ValueError(
+                        f"Encountered error while importing {package_name}: {e}. {_get_mmlab_installation_guide(package_name)}"
+                    )
+            else:
+                raise ValueError(f"package_name {package_name} is not required.")
+
+
+def _get_mmlab_installation_guide(package_name):
+    if package_name == "mmdet":
+        err_msg = 'Please install MMDetection by: pip install "mmdet==3.2.0"'
+    elif package_name == "mmcv":
+        err_msg = 'Please install MMCV by: mim install "mmcv==2.1.0"'
+    elif package_name == "mmengine":
+        err_msg = "Please install MMEngine by: mim install mmengine"
+    else:
+        raise ValueError("Available package_name are: mmdet, mmcv, mmengine.")
+
+    err_msg += " Pytorch version larger than 2.1 is not supported yet. To use Autogluon for object detection, please downgrade PyTorch version to <=2.1."
+
+    return err_msg

autogluon/multimodal/utils/load.py

@@ -1,59 +1,13 @@
 import logging
 import os
 import pickle
-from typing import Dict, List, Optional, Tuple, Union
+import zipfile
 
 from ..constants import LAST_CHECKPOINT, MODEL_CHECKPOINT
-from ..data import DocumentProcessor, NerProcessor, TextProcessor
-from ..models.utils import get_pretrained_tokenizer
 
 logger = logging.getLogger(__name__)
 
 
-def load_text_tokenizers(
-    text_processors: Union[List[TextProcessor], List[NerProcessor], List[DocumentProcessor]],
-    path: str,
-) -> Union[List[TextProcessor], List[NerProcessor], List[DocumentProcessor]]:
-    """
-    Load saved text tokenizers. If text/ner processors already have tokenizers,
-    then do nothing.
-
-    Parameters
-    ----------
-    text_processors
-        A list of text/ner processors with tokenizers or their relative paths.
-    path
-        The root path.
-
-    Returns
-    -------
-    A list of text/ner processors with tokenizers loaded.
-    """
-    for per_text_processor in text_processors:
-        if isinstance(per_text_processor.tokenizer, str):
-            per_path = os.path.join(path, per_text_processor.tokenizer)
-            per_text_processor.tokenizer = get_pretrained_tokenizer(
-                tokenizer_name=per_text_processor.tokenizer_name,
-                checkpoint_name=per_path,
-            )
-    return text_processors
-
-
-class CustomUnpickler(pickle.Unpickler):
-    """
-    This is to make pickle loading df_preprocessor backward compatible.
-    A df_preprocessor object saved with old name space `autogluon.text.automm` has errors
-    when being loaded under the context of new name `autogluon.multimodal`.
-    """
-
-    def find_class(self, module, name):
-        renamed_module = module
-        if module.startswith("autogluon.text.automm"):
-            renamed_module = module.replace("autogluon.text.automm", "autogluon.multimodal")
-
-        return super(CustomUnpickler, self).find_class(renamed_module, name)
-
-
 def get_dir_ckpt_paths(path: str):
     """
     Get the dir path and ckpt path from a path.
@@ -138,3 +92,54 @@ def get_load_ckpt_paths(ckpt_path: str, dir_path: str, resume: bool):
             ckpt_path = None  # must set None since we do not resume training
 
     return load_path, ckpt_path
+
+
+class CustomUnpickler(pickle.Unpickler):
+    """
+    This is to make pickle loading an object backward compatible.
+    A df_preprocessor object saved with old name space `xxx.yyy` has errors
+    when being loaded under the context of new name `aaa.bbb`.
+    """
+
+    def find_class(self, module, name):
+        renamed_module = module
+        if module.startswith("autogluon.text.automm"):
+            renamed_module = module.replace("autogluon.text.automm", "autogluon.multimodal")
+
+        return super(CustomUnpickler, self).find_class(renamed_module, name)
+
+
+def protected_zip_extraction(zipfile_path, sha1_hash, folder):
+    """
+    Extract zip file to the folder.
+
+    A signature file named ".SHA1HASH.sig" will be created if the extraction has been finished.
+
+    Returns
+    -------
+    folder
+        The directory to extract the zipfile
+    """
+    os.makedirs(folder, exist_ok=True)
+
+    if sha1_hash:
+        sha1_hash = sha1_hash[:6]
+        signature = ".{}.sig".format(sha1_hash)
+
+        if os.path.exists(os.path.join(folder, signature)):
+            # We have found the signature file. Thus, we will not extract again.
+            return folder
+    else:
+        signature = None
+
+    # Extract the file
+    logging.info("Extract files...")
+    with zipfile.ZipFile(zipfile_path, "r") as zip_ref:
+        zip_ref.extractall(folder)
+
+    if signature:
+        # Create the signature
+        with open(os.path.join(folder, signature), "w"):
+            pass
+
+    return folder

autogluon/multimodal/utils/log.py

@@ -4,52 +4,13 @@ import os
 from contextlib import contextmanager
 from typing import Dict, List, Optional, Tuple, Union
 
-import pytz
 import torch
 
 from autogluon.common.utils.system_info import get_ag_system_info
 
-logger = logging.getLogger(__name__)
-
-
-def make_exp_dir(
-    root_path: str,
-    job_name: str,
-    create: Optional[bool] = True,
-):
-    """
-    Creates the exp dir of format e.g.,: root_path/2022_01_01/job_name_12_00_00/
-    This function is to better organize the training runs. It is recommended to call this
-    function and pass the returned "exp_dir" to "MultiModalPredictor.fit(save_path=exp_dir)".
-
-    Parameters
-    ----------
-    root_path
-        The basic path where to create saving directories for training runs.
-    job_name
-        The job names to name training runs.
-    create
-        Whether to make the directory.
-
-    Returns
-    -------
-    The formatted directory path.
-    """
-    tz = pytz.timezone("US/Pacific")
-    ct = datetime.datetime.now(tz=tz)
-    date_stamp = ct.strftime("%Y_%m_%d")
-    time_stamp = ct.strftime("%H_%M_%S")
-
-    # Group logs by day first
-    exp_dir = os.path.join(root_path, date_stamp)
-
-    # Then, group by run_name and hour + min + sec to avoid duplicates
-    exp_dir = os.path.join(exp_dir, "_".join([job_name, time_stamp]))
-
-    if create:
-        os.makedirs(exp_dir, mode=0o777, exist_ok=False)
+from .strategy import is_interactive_strategy
 
-    return exp_dir
+logger = logging.getLogger(__name__)
 
 
 class LogFilter(logging.Filter):
@@ -200,6 +161,10 @@ def get_gpu_message(detected_num_gpus: int, used_num_gpus: int, strategy: str):
         return round((bytes / 1024) / 1024 / 1024, 2)
 
     gpu_message = f"GPU Count: {detected_num_gpus}\nGPU Count to be Used: {used_num_gpus}\n"
+
+    if is_interactive_strategy(strategy):  # avoid pre-initializing cuda when using ddp_fork
+        return gpu_message
+
     try:
         import nvidia_smi
     except:

autogluon/multimodal/utils/matcher.py

@@ -11,8 +11,8 @@ from torch import nn
 from torch.nn import functional as F
 
 from ..constants import FUSION, QUERY, RESPONSE
-from .data import data_to_df
-from .model import create_model
+from ..data import data_to_df
+from ..models import create_model
 
 logger = logging.getLogger(__name__)
 
@@ -103,6 +103,7 @@ def create_fusion_model_dict(
             model_name=model_name,
             model_config=model_config,
             pretrained=pretrained,
+            is_matching=True,  # clip needs to use this to init attributes for both image and text
         )
         if model_name.lower().startswith(FUSION):
             fusion_model = model

autogluon/multimodal/utils/onnx.py

@@ -2,10 +2,6 @@ import logging
 import os
 from typing import Dict, List, Optional, Tuple, Union
 
-from torch import tensor
-
-from ..constants import AUTOMM, FEATURE_EXTRACTION, MULTICLASS
-
 logger = logging.getLogger(__name__)
 
 # TODO: Try a better workaround to lazy import tensorrt package.

autogluon/multimodal/utils/path.py

@@ -0,0 +1,10 @@
+import logging
+import os
+from datetime import datetime
+from typing import Dict, List, Optional, Tuple, Union
+
+import pytz
+
+from ..constants import LAST_CHECKPOINT, MODEL_CHECKPOINT
+
+logger = logging.getLogger(__name__)