autogluon.multimodal 1.2.1b20250303__py3-none-any.whl → 1.2.1b20250305__py3-none-any.whl

autogluon/multimodal/optim/metrics/utils.py

@@ -0,0 +1,359 @@
+import functools
+import logging
+import warnings
+from typing import Dict, List, Optional, Tuple, Union
+
+import evaluate
+import torchmetrics
+from torch.nn import functional as F
+from torchmetrics.detection.mean_ap import MeanAveragePrecision
+
+from autogluon.core.metrics import Scorer, compute_metric, get_metric
+
+from ...constants import (
+    ACC,
+    ACCURACY,
+    AVERAGE_PRECISION,
+    BER,
+    COSINE_EMBEDDING_LOSS,
+    COVERAGE,
+    CROSS_ENTROPY,
+    DETECTION_METRICS,
+    DIRECT_LOSS,
+    EM,
+    F1,
+    F1_MACRO,
+    F1_MICRO,
+    F1_WEIGHTED,
+    FM,
+    HIT_RATE,
+    IOU,
+    LOG_LOSS,
+    MAE,
+    MATCHING_METRICS,
+    MATCHING_METRICS_WITHOUT_PROBLEM_TYPE,
+    MAX,
+    METRIC_MODE_MAP,
+    MIN,
+    MULTICLASS,
+    NER_TOKEN_F1,
+    OVERALL_ACCURACY,
+    OVERALL_F1,
+    PEARSONR,
+    QUADRATIC_KAPPA,
+    R2,
+    RECALL,
+    RETRIEVAL_METRICS,
+    RMSE,
+    ROC_AUC,
+    ROOT_MEAN_SQUARED_ERROR,
+    SM,
+    SPEARMANR,
+    Y_PRED,
+    Y_PRED_PROB,
+    Y_TRUE,
+)
+from .coverage_metrics import Coverage
+from .hit_rate_metrics import CustomHitRate
+from .semantic_seg_metrics import COD_METRICS_NAMES, Balanced_Error_Rate, Binary_IoU, Multiclass_IoU
+
+logger = logging.getLogger(__name__)
+
+
+def compute_score(
+    metric_data: dict,
+    metric: Union[str, Scorer],
+    pos_label: Optional[int] = 1,
+) -> float:
+    """
+    Use sklearn to compute the score of one metric.
+
+    Parameters
+    ----------
+    metric_data
+        A dictionary with the groundtruth (Y_TRUE) and predicted values (Y_PRED, Y_PRED_PROB).
+        The predicted class probabilities are required to compute the roc_auc score.
+    metric
+        The name of metric or the function of metric to compute.
+    pos_label
+        The encoded label (0 or 1) of binary classification's positive class.
+
+    Returns
+    -------
+    Computed score.
+    """
+    if isinstance(metric, str) and metric in [OVERALL_ACCURACY, OVERALL_F1]:
+        metric = evaluate.load("seqeval")
+        warnings.filterwarnings("ignore")
+        for p in metric_data[Y_TRUE]:
+            if "_" in p:
+                print(p)
+        for p in metric_data[Y_PRED]:
+            if "_" in p:
+                print(p)
+        return metric.compute(references=metric_data[Y_TRUE], predictions=metric_data[Y_PRED])
+
+    metric = get_metric(metric)
+
+    y = metric_data[Y_TRUE]
+    if metric.needs_proba or metric.needs_threshold:
+        y_pred_proba = metric_data[Y_PRED_PROB]
+        y_pred_proba = (
+            y_pred_proba if y_pred_proba.shape[1] > 2 else y_pred_proba[:, pos_label]
+        )  # only use pos_label for binary classification
+        return metric.convert_score_to_original(
+            compute_metric(y=y, y_pred_proba=y_pred_proba, metric=metric, weights=None)
+        )
+    else:
+        y_pred = metric_data[Y_PRED]
+
+        # TODO: This is a hack. Doesn't support `f1_macro`, `f1_micro`, `f1_weighted`, or custom `f1` metrics with different names.
+        # TODO: Longterm the solution should be to have the input data to this function use the internal representation without the original class names. This way `pos_label` would not need to be specified.
+        if metric.name == F1:  # only for binary classification
+            y = (y == pos_label).astype(int)
+            y_pred = (y_pred == pos_label).astype(int)
+
+        return metric.convert_score_to_original(compute_metric(y=y, y_pred=y_pred, metric=metric, weights=None))
+
+
+def infer_metrics(
+    problem_type: Optional[str] = None,
+    eval_metric: Optional[Union[str, Scorer]] = None,
+    validation_metric_name: Optional[str] = None,
+    is_matching: Optional[bool] = False,
+):
+    """
+    Infer the validation metric and the evaluation metric if not provided.
+    Validation metric is for early-stopping and selecting the best model checkpoints.
+    Evaluation metric is to report performance to users.
+
+    Parameters
+    ----------
+    problem_type
+        Type of problem.
+    eval_metric
+        Name of evaluation metric provided by users.
+    validation_metric_name
+        The provided validation metric name
+    is_matching
+        Whether is matching.
+
+    Returns
+    -------
+    validation_metric_name
+        Name of validation metric.
+    eval_metric_name
+        Name of evaluation metric.
+    """
+    is_customized = False
+    if eval_metric is None:
+        eval_metric_name = None
+    elif isinstance(eval_metric, str):
+        eval_metric_name = eval_metric
+    elif isinstance(eval_metric, Scorer):
+        eval_metric_name = eval_metric.name
+        is_customized = True
+    else:
+        raise TypeError(f"eval_metric can be a str, a Scorer, or None, but is type: {type(eval_metric)}")
+
+    if problem_type is not None:
+        from ...utils.problem_types import PROBLEM_TYPES_REG
+
+        problem_property = PROBLEM_TYPES_REG.get(problem_type)
+
+    if is_matching:
+        if eval_metric_name is not None:
+            # if eval_metric_name is a valid metric
+            if eval_metric_name.lower() in METRIC_MODE_MAP.keys():
+                validation_metric_name = eval_metric_name
+                return validation_metric_name, eval_metric_name
+            elif eval_metric_name.lower() in RETRIEVAL_METRICS:
+                # Currently only support recall as validation metric in retrieval tasks.
+                validation_metric_name = RECALL
+                return validation_metric_name, eval_metric_name
+
+        # When eval_metric_name is either None or not supported:
+        # Fallback based on problem type unless it's a customized metric
+        if problem_type is None:
+            validation_metric_name, fallback_evaluation_metric = MATCHING_METRICS_WITHOUT_PROBLEM_TYPE
+        elif problem_type in MATCHING_METRICS:
+            validation_metric_name, fallback_evaluation_metric = MATCHING_METRICS[problem_type]
+        else:
+            raise NotImplementedError(f"Problem type: {problem_type} is not yet supported for matching!")
+        if not is_customized:
+            if eval_metric_name is not None:
+                warnings.warn(
+                    f"Metric {eval_metric_name} is not supported as the evaluation metric for {problem_type} in matching tasks."
+                    f"The evaluation metric is changed to {fallback_evaluation_metric} by default."
+                )
+            eval_metric_name = fallback_evaluation_metric
+        return validation_metric_name, eval_metric_name
+
+    if eval_metric_name is not None:
+        # Infer evaluation metric
+        if eval_metric_name.lower() not in problem_property.supported_evaluation_metrics and not is_customized:
+            warnings.warn(
+                f"Metric {eval_metric_name} is not supported as the evaluation metric for {problem_type}. "
+                f"The evaluation metric is changed to {problem_property.fallback_evaluation_metric} by default."
+            )
+            if problem_property.fallback_evaluation_metric is not None:
+                eval_metric_name = problem_property.fallback_evaluation_metric
+            else:
+                # Problem types like extract_embedding does not need a eval/val metric
+                return None, None
+
+        # Infer validation metric
+        if eval_metric_name.lower() in problem_property.supported_validation_metrics:
+            validation_metric_name = eval_metric_name
+        else:
+            if problem_property.fallback_validation_metric is not None:
+                validation_metric_name = problem_property.fallback_validation_metric
+    else:
+        eval_metric_name = problem_property.fallback_evaluation_metric
+        validation_metric_name = problem_property.fallback_validation_metric
+
+    return validation_metric_name, eval_metric_name
+
+
+def get_minmax_mode(
+    metric_name: Union[str, Scorer],
+):
+    """
+    Get minmax mode based on metric name
+
+    Parameters
+    ----------
+    metric_name
+        A string representing metric
+
+    Returns
+    -------
+    mode
+        The min/max mode used in selecting model checkpoints.
+        - min
+             Its means that smaller metric is better.
+        - max
+            It means that larger metric is better.
+    """
+    if isinstance(metric_name, str):
+        assert (
+            metric_name in METRIC_MODE_MAP
+        ), f"{metric_name} is not a supported metric. Options are: {METRIC_MODE_MAP.keys()}"
+        return METRIC_MODE_MAP.get(metric_name)
+    else:
+        return MAX if metric_name.greater_is_better else MIN
+
+
+def get_stopping_threshold(metric_name: str):
+    """
+    Get the metric threshold for early stopping.
+
+    Parameters
+    ----------
+    metric_name
+        Name of validation metric.
+
+    Returns
+    -------
+    The stopping threshold.
+    """
+    try:
+        metric = get_metric(metric_name)
+        stopping_threshold = metric.optimum - metric._sign * 1e-7
+    except:
+        stopping_threshold = None
+
+    return stopping_threshold
+
+
+def get_torchmetric(
+    metric_name: str,
+    num_classes: Optional[int] = None,
+    is_matching: Optional[bool] = False,
+    problem_type: Optional[str] = None,
+):
+    """
+    Obtain a torchmerics.Metric from its name.
+    Define a customized metric function in case that torchmetrics doesn't support some metric.
+
+    Parameters
+    ----------
+    metric_name
+        Name of metric.
+    num_classes
+        Number of classes.
+    is_matching
+        Whether is matching.
+    problem_type
+        Type of problem, e.g., binary and multiclass.
+
+    Returns
+    -------
+    torchmetrics.Metric
+        A torchmetrics.Metric object.
+    custom_metric_func
+        A customized metric function.
+    """
+    metric_name = metric_name.lower()
+    if metric_name in [ACC, ACCURACY, OVERALL_ACCURACY]:
+        # use MULTICLASS since the head output dim is 2 for the binary problem type.
+        return torchmetrics.Accuracy(task=MULTICLASS, num_classes=num_classes), None
+    elif metric_name == NER_TOKEN_F1:
+        return torchmetrics.F1Score(task=MULTICLASS, num_classes=num_classes, ignore_index=1), None
+    elif metric_name in [RMSE, ROOT_MEAN_SQUARED_ERROR]:
+        return torchmetrics.MeanSquaredError(squared=False), None
+    elif metric_name == R2:
+        return torchmetrics.R2Score(), None
+    elif metric_name == QUADRATIC_KAPPA:
+        return (
+            torchmetrics.CohenKappa(task=problem_type, num_classes=num_classes, weights="quadratic"),
+            None,
+        )
+    elif metric_name == ROC_AUC:
+        return torchmetrics.AUROC(task=problem_type, num_classes=num_classes), None
+    elif metric_name == AVERAGE_PRECISION:
+        return torchmetrics.AveragePrecision(task=problem_type, num_classes=num_classes)
+    elif metric_name in [LOG_LOSS, CROSS_ENTROPY]:
+        return torchmetrics.MeanMetric(), functools.partial(F.cross_entropy, reduction="none")
+    elif metric_name == COSINE_EMBEDDING_LOSS:
+        return torchmetrics.MeanMetric(), functools.partial(F.cosine_embedding_loss, reduction="none")
+    elif metric_name == PEARSONR:
+        return torchmetrics.PearsonCorrCoef(), None
+    elif metric_name == SPEARMANR:
+        if is_matching:  # TODO: add support for matching.
+            raise ValueError("spearman relation is not supported for matching yet.")
+        else:
+            return torchmetrics.SpearmanCorrCoef(), None
+    elif metric_name == F1:
+        return torchmetrics.F1Score(task=problem_type, num_classes=num_classes), None
+    elif metric_name in [F1_MACRO, F1_MICRO, F1_WEIGHTED]:
+        average = metric_name.split("_")[1]
+        return torchmetrics.F1Score(task=problem_type, num_classes=num_classes, average=average), None
+    elif metric_name in DETECTION_METRICS:
+        return (
+            MeanAveragePrecision(box_format="xyxy", iou_type="bbox", class_metrics=False),
+            None,
+        )  # TODO: remove parameter hardcodings here, and add class_metrics
+    elif metric_name == DIRECT_LOSS:
+        return (
+            torchmetrics.MeanMetric(nan_strategy="warn"),
+            None,
+        )  # This only works for detection where custom_metric is not required for BaseAggregator
+    elif metric_name in [RECALL, HIT_RATE]:
+        if is_matching:
+            return CustomHitRate(), None
+        else:  # TODO: support recall for general classification tasks.
+            raise ValueError("Recall is not supported yet.")
+    elif metric_name == BER:
+        return Balanced_Error_Rate(), None
+    elif metric_name in [SM, EM, FM, MAE]:
+        return COD_METRICS_NAMES[metric_name], None
+    elif metric_name == IOU:
+        if num_classes == 1:
+            return Binary_IoU(), None
+        else:
+            return Multiclass_IoU(num_classes=num_classes), None
+    elif metric_name == COVERAGE:
+        return Coverage(), None
+    else:
+        raise ValueError(f"Unknown metric {metric_name}")

autogluon/multimodal/optim/utils.py

@@ -0,0 +1,284 @@
+import logging
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import torch
+from torch import nn, optim
+from transformers import Adafactor
+from transformers.trainer_pt_utils import get_parameter_names
+
+from ..constants import (
+    BIT_FIT,
+    COLUMN_FEATURES,
+    CONV_LORA,
+    FEATURES,
+    IA3,
+    IA3_BIAS,
+    IA3_LORA,
+    IA3_LORA_BIAS,
+    IA3_LORA_NORM,
+    IA3_NORM,
+    LORA,
+    LORA_BIAS,
+    LORA_NORM,
+    NORM_FIT,
+    PEFT_STRATEGIES,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_optimizer(
+    optim_type: str,
+    optimizer_grouped_parameters,
+    lr: float,
+    weight_decay: float,
+    eps: Optional[float] = 1e-6,
+    betas: Optional[Tuple[float, float]] = (0.9, 0.999),
+    momentum: Optional[float] = 0.9,
+):
+    """
+    Choose a Pytorch optimizer based on its name.
+
+    Parameters
+    ----------
+    optim_type
+        Name of optimizer.
+    optimizer_grouped_parameters
+        The model parameters to be optimized.
+    lr
+        Learning rate.
+    weight_decay
+        Optimizer weight decay.
+    eps
+        Optimizer eps.
+    betas
+        Optimizer betas.
+    momentum
+        Momentum used in the SGD optimizer.
+
+    Returns
+    -------
+    A Pytorch optimizer.
+    """
+    if optim_type == "adamw":
+        optimizer = optim.AdamW(
+            optimizer_grouped_parameters,
+            lr=lr,
+            weight_decay=weight_decay,
+            eps=eps,
+            betas=betas,
+        )
+    elif optim_type == "adam":
+        optimizer = optim.Adam(
+            optimizer_grouped_parameters,
+            lr=lr,
+            weight_decay=weight_decay,
+        )
+    elif optim_type == "sgd":
+        optimizer = optim.SGD(
+            optimizer_grouped_parameters,
+            lr=lr,
+            weight_decay=weight_decay,
+            momentum=momentum,
+        )
+    elif optim_type == "adafactor":
+        optimizer = Adafactor(
+            optimizer_grouped_parameters,
+            lr=lr,
+            weight_decay=weight_decay,
+            scale_parameter=True,  # Generally recommended to enable scaling
+            relative_step=False,
+            warmup_init=False,
+        )
+    else:
+        raise ValueError(f"unknown optimizer: {optim_type}")
+
+    return optimizer
+
+
+def get_weight_decay_param_names(model: nn.Module):
+    """
+    Set the layer normalization parameters and other layers' bias parameters not to use weight decay.
+
+    Parameters
+    ----------
+    model
+        A Pytorch model.
+
+    Returns
+    -------
+    A list of parameter names not using weight decay.
+    """
+    # By default, we should not apply weight decay for all the norm layers
+    decay_param_names = get_parameter_names(
+        model,
+        [nn.LayerNorm, nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.GroupNorm],
+    )
+    decay_param_names = [
+        name
+        for name in decay_param_names
+        if (
+            "bias" not in name
+            and "cls_token" not in name
+            and "categorical_feature_tokenizer" not in name
+            and "numerical_feature_tokenizer" not in name
+        )
+    ]
+    return decay_param_names
+
+
+def get_norm_layer_param_names(model: nn.Module):
+    """
+    Get parameters associated with the normalization layers
+
+    Parameters
+    ----------
+    model
+        A Pytorch model
+
+    Returns
+    -------
+    norm_param_names
+        A list of normalization parameter names
+    """
+    all_param_names = [name for name, _ in model.named_parameters()]
+    all_param_names_except_norm_names = get_parameter_names(
+        model,
+        [nn.LayerNorm, nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.GroupNorm],
+    )
+    norm_param_names = [name for name in all_param_names if name not in all_param_names_except_norm_names]
+    return norm_param_names
+
+
+def get_peft_param_names(norm_param_names: List[str], peft: Optional[str] = None, extra_params: Optional[List] = None):
+    """
+     Get the list of trainable parameters according to the provided efficient finetuning method.
+
+    Parameters
+    ----------
+    norm_param_names
+        The parameters associated with the normalization layers
+    peft
+        Efficient finetuning strategy. Trainable parameters will be adjusted according to the method.
+    extra_params
+        Extra parameters to train.
+
+    Returns
+    -------
+    Get list of trainable parameter names according to the provided efficient finetuning method.
+    """
+    peft_param_names = []
+
+    if peft == BIT_FIT:
+        peft_param_names.append(".*bias*.")
+    elif peft == NORM_FIT:
+        peft_param_names.append(".*bias*.")
+        peft_param_names += norm_param_names
+    elif peft in [LORA, IA3, IA3_LORA, CONV_LORA]:
+        peft_param_names.append(".*lora_*.")
+    elif peft in [LORA_BIAS, IA3_BIAS, IA3_LORA_BIAS]:
+        peft_param_names.append(".*lora_*.")
+        peft_param_names.append(".*bias*.")
+    elif peft in [LORA_NORM, IA3_NORM, IA3_LORA_NORM]:
+        peft_param_names.append(".*lora_*.")
+        peft_param_names.append(".*bias*.")
+        peft_param_names += norm_param_names
+    elif peft is not None:
+        raise NotImplementedError(
+            f"The efficient finetuning strategy '{peft}'"
+            f" is not supported. We only support"
+            f" {', '.join(PEFT_STRATEGIES)}."
+        )
+
+    if extra_params:
+        peft_param_names.extend(extra_params)
+
+    return peft_param_names
+
+
+def remove_parameters_without_grad(
+    grouped_parameters: List[Dict],
+):
+    """
+    Remove layers
+
+    Parameters
+    ----------
+    grouped_parameters
+        The grouped parameters or their names output from lr_choice.
+
+    Returns
+    -------
+    The updated grouped parameters or their names.
+    """
+    for group_idx, group_param in enumerate(grouped_parameters):
+        updated_params = []
+        for p in group_param["params"]:
+            if p.requires_grad:
+                updated_params.append(p)
+        grouped_parameters[group_idx]["params"] = updated_params
+
+    return grouped_parameters
+
+
+def gather_column_features(
+    output: Dict[str, Dict],
+    column_names: Union[str, List[str]],
+):
+    """
+    Gather column features from models' outputs.
+    For each feature name in one model's output, we enumerate the provided column names to see
+    whether (partial) the provided columns share one cls feature or they have independent features.
+
+    TODO: return features' masks and use them to filter the losses.
+
+    Parameters
+    ----------
+    output
+        The models' outputs.
+    column_names
+        The columns whose features we want to get.
+
+    Returns
+    -------
+    The gathered feature vectors. Each sample should only have one feature vector.
+    """
+    if isinstance(column_names, str):
+        column_names = [column_names]
+
+    gathered_features = []
+    # logger.debug(f"gather features for columns: {column_names}")
+    for per_model_name, per_model_output in output.items():
+        # logger.debug(f"gather column features from model: {per_model_name}")
+        for feature_name in per_model_output[COLUMN_FEATURES][FEATURES]:
+            # logger.debug(f"processing feature: {feature_name}")
+            columns_share_one_feature = []
+            for col_name in column_names:
+                if col_name in feature_name:
+                    # this column feature is part of the cls feature
+                    if not (feature_name.startswith(col_name) and feature_name.endswith(col_name)):
+                        columns_share_one_feature.append(col_name)
+                        # logger.debug(f"column {col_name} is included in feature {feature_name}")
+                    else:  # this column's feature is independent of other columns'
+                        gathered_features.append(per_model_output[COLUMN_FEATURES][FEATURES][col_name])
+                        # logger.debug(f"col_name {col_name} has an independent feature in model: {per_model_name}")
+
+            # two or more columns share one cls feature, and no other columns share it.
+            if len(columns_share_one_feature) > 0:
+                assert (
+                    len("_".join(columns_share_one_feature)) == len(feature_name)
+                ), f"model `{per_model_name}`'s cls feature name `{feature_name}` doesn't match `{columns_share_one_feature}`"
+                gathered_features.append(per_model_output[COLUMN_FEATURES][FEATURES][feature_name])
+
+    if len(gathered_features) > 1:
+        # currently only support features of the same shape
+        assert all(
+            per_features.shape == gathered_features[0].shape for per_features in gathered_features
+        ), "Currently we only support gathering features of the same dimension."
+
+    if len(gathered_features) == 0:
+        raise ValueError(f"No features are found for columns names {column_names}.")
+
+    gathered_features = torch.stack(gathered_features, dim=0).mean(dim=0)  # (b, d)
+
+    return gathered_features