autogluon.multimodal 1.1.2b20241111__py3-none-any.whl → 1.1.2b20241113__py3-none-any.whl

autogluon/multimodal/constants.py

@@ -177,6 +177,23 @@ MATCHING_METRICS = {
 }
 MATCHING_METRICS_WITHOUT_PROBLEM_TYPE = [RECALL, NDCG]
 
+EVALUATION_METRICS = {
+    # Use evaluation metrics from METRICS for these types
+    BINARY: METRICS[BINARY].keys(),
+    MULTICLASS: METRICS[MULTICLASS].keys(),
+    REGRESSION: METRICS[REGRESSION].keys(),
+    OBJECT_DETECTION: DETECTION_METRICS,
+    SEMANTIC_SEGMENTATION: [IOU, BER, SM],
+    NER: [OVERALL_F1, NER_TOKEN_F1],
+    NAMED_ENTITY_RECOGNITION: [OVERALL_F1, NER_TOKEN_F1],
+    FEW_SHOT_CLASSIFICATION: METRICS[MULTICLASS].keys(),
+}
+
+VALIDATION_METRICS = {
+    problem_type: [metric for metric in metrics if metric in METRIC_MODE_MAP] + [DIRECT_LOSS]
+    for problem_type, metrics in EVALUATION_METRICS.items()
+}
+
 # Training status
 TRAIN = "train"
 VALIDATE = "validate"

autogluon/multimodal/problem_types.py

@@ -3,16 +3,12 @@
 from dataclasses import dataclass, field
 from typing import List, Optional, Set
 
-from autogluon.core.metrics import METRICS
-
 from .constants import (
     ACCURACY,
-    BER,
     BINARY,
     CATEGORICAL,
     CLASSIFICATION,
-    DETECTION_METRICS,
-    DIRECT_LOSS,
+    EVALUATION_METRICS,
     FEATURE_EXTRACTION,
     FEW_SHOT_CLASSIFICATION,
     IMAGE,
@@ -22,7 +18,6 @@ from .constants import (
     IMAGE_TEXT_SIMILARITY,
     IOU,
     MAP,
-    METRIC_MODE_MAP,
     MULTICLASS,
     NAMED_ENTITY_RECOGNITION,
     NER,
@@ -30,16 +25,15 @@ from .constants import (
     NER_TOKEN_F1,
     NUMERICAL,
     OBJECT_DETECTION,
-    OVERALL_F1,
     REGRESSION,
     RMSE,
     ROC_AUC,
     ROIS,
     SEMANTIC_SEGMENTATION,
-    SM,
     TEXT,
     TEXT_NER,
     TEXT_SIMILARITY,
+    VALIDATION_METRICS,
     ZERO_SHOT_IMAGE_CLASSIFICATION,
 )
 from .registry import Registry
@@ -76,11 +70,10 @@ class ProblemTypeProperty:
     # The fallback label type of the problem
     _fallback_label_type: Optional[str] = None
 
-    # The evaluation metrics supported
-    # The FIRST metric is used as fallback by default
-    _supported_evaluation_metrics: Optional[List[str]] = None
+    # If evaluations are supported
+    _support_eval: Optional[bool] = False
 
-    # Overwrite the default setting (first in _supported_evaluation_metrics)
+    # Overwrite the default setting (first in supported_evaluation_metrics)
     _fallback_evaluation_metric: Optional[str] = None
 
     # The validation metric fallback
@@ -96,8 +89,8 @@ class ProblemTypeProperty:
 
     @property
     def supported_evaluation_metrics(self):
-        if self._supported_evaluation_metrics:
-            return self._supported_evaluation_metrics
+        if self._support_eval:
+            return EVALUATION_METRICS[self.name]
         else:
             return []
 
@@ -105,17 +98,15 @@ class ProblemTypeProperty:
     def fallback_evaluation_metric(self):
         if self._fallback_evaluation_metric:
             return self._fallback_evaluation_metric
-        elif self._supported_evaluation_metrics:
-            return self._supported_evaluation_metrics[0]
+        elif self._support_eval:
+            return self.supported_evaluation_metrics[0]
         else:
             return None
 
     @property
     def supported_validation_metrics(self):
-        if self._supported_evaluation_metrics:
-            return [metric for metric in self._supported_evaluation_metrics if metric in METRIC_MODE_MAP.keys()] + [
-                DIRECT_LOSS
-            ]
+        if self._support_eval:
+            return VALIDATION_METRICS[self.name]
         else:
             return []
 
@@ -145,7 +136,7 @@ PROBLEM_TYPES_REG.register(
         supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},
         supported_label_type={CATEGORICAL},
         is_classification=True,
-        _supported_evaluation_metrics=METRICS[BINARY].keys(),
+        _support_eval=True,
         _fallback_evaluation_metric=ROC_AUC,
         _fallback_validation_metric=ROC_AUC,
     ),
@@ -157,7 +148,7 @@ PROBLEM_TYPES_REG.register(
         supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},
         supported_label_type={CATEGORICAL},
         is_classification=True,
-        _supported_evaluation_metrics=METRICS[MULTICLASS].keys(),
+        _support_eval=True,
         _fallback_evaluation_metric=ACCURACY,
         _fallback_validation_metric=ACCURACY,
     ),
@@ -170,7 +161,7 @@ PROBLEM_TYPES_REG.register(
         name=REGRESSION,
         supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},
         supported_label_type={NUMERICAL},
-        _supported_evaluation_metrics=METRICS[REGRESSION].keys(),
+        _support_eval=True,
         _fallback_evaluation_metric=RMSE,
         _fallback_validation_metric=RMSE,
     ),
@@ -185,7 +176,7 @@ PROBLEM_TYPES_REG.register(
         supported_modality_type={IMAGE},
         supported_label_type={ROIS},
         force_exist_modality={IMAGE},
-        _supported_evaluation_metrics=DETECTION_METRICS,
+        _support_eval=True,
         _fallback_validation_metric=MAP,
     ),
 )
@@ -200,7 +191,7 @@ PROBLEM_TYPES_REG.register(
         supported_modality_type={IMAGE},
         supported_label_type={IMAGE},
         force_exist_modality={IMAGE},
-        _supported_evaluation_metrics=[IOU, BER, SM],
+        _support_eval=True,
         _fallback_evaluation_metric=IOU,
         _fallback_validation_metric=IOU,
     ),
@@ -247,7 +238,7 @@ _ner_property = ProblemTypeProperty(
     supported_modality_type={IMAGE, TEXT, CATEGORICAL, NUMERICAL, TEXT_NER},
     supported_label_type={NER_ANNOTATION},
     force_exist_modality={TEXT_NER},
-    _supported_evaluation_metrics=[OVERALL_F1, NER_TOKEN_F1],
+    _support_eval=True,
     _fallback_validation_metric=NER_TOKEN_F1,
 )
 PROBLEM_TYPES_REG.register(NER, _ner_property)
@@ -281,7 +272,7 @@ PROBLEM_TYPES_REG.register(
         support_zero_shot=False,
         supported_modality_type={IMAGE, TEXT},
         supported_label_type={CATEGORICAL},
-        _supported_evaluation_metrics=METRICS[MULTICLASS].keys(),
+        _support_eval=True,
         _fallback_evaluation_metric=ACCURACY,
         _fallback_validation_metric=ACCURACY,
     ),

autogluon/multimodal/utils/metric.py

@@ -8,7 +8,7 @@ import evaluate
 import numpy as np
 from sklearn.metrics import f1_score
 
-from autogluon.core.metrics import Scorer, get_metric
+from autogluon.core.metrics import Scorer, compute_metric, get_metric
 
 from ..constants import (
     ACCURACY,
@@ -231,15 +231,22 @@ def compute_score(
         return metric.compute(references=metric_data[Y_TRUE], predictions=metric_data[Y_PRED])
 
     metric = get_metric(metric)
-    if metric.name in [ROC_AUC, AVERAGE_PRECISION]:
-        return metric._sign * metric(metric_data[Y_TRUE], metric_data[Y_PRED_PROB][:, pos_label])
-    elif metric.name in [F1]:  # only for binary classification
-        return f1_score(metric_data[Y_TRUE], metric_data[Y_PRED], pos_label=pos_label)
+
+    y = metric_data[Y_TRUE]
+    if metric.needs_proba or metric.needs_threshold:
+        return metric.convert_score_to_original(
+            compute_metric(y=y, y_pred_proba=metric_data[Y_PRED_PROB][:, pos_label], metric=metric, weights=None)
+        )
     else:
-        try:
-            return metric._sign * metric(metric_data[Y_TRUE], metric_data[Y_PRED], y_prob=metric_data[Y_PRED_PROB])
-        except:
-            return metric._sign * metric(metric_data[Y_TRUE], metric_data[Y_PRED])
+        y_pred = metric_data[Y_PRED]
+
+        # TODO: This is a hack. Doesn't support `f1_macro`, `f1_micro`, `f1_weighted`, or custom `f1` metrics with different names.
+        # TODO: Longterm the solution should be to have the input data to this function use the internal representation without the original class names. This way `pos_label` would not need to be specified.
+        if metric.name == F1:  # only for binary classification
+            y = (metric_data[Y_TRUE] == pos_label).astype(int)
+            y_pred = (metric_data[Y_PRED] == pos_label).astype(int)
+
+        return metric.convert_score_to_original(compute_metric(y=y, y_pred=y_pred, metric=metric, weights=None))
 
 
 class RankingMetrics:

autogluon/multimodal/version.py

@@ -1,3 +1,3 @@
 """This is the autogluon version file."""
-__version__ = '1.1.2b20241111'
+__version__ = '1.1.2b20241113'
 __lite__ = False

{autogluon.multimodal-1.1.2b20241111.dist-info → autogluon.multimodal-1.1.2b20241113.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: autogluon.multimodal
-Version: 1.1.2b20241111
+Version: 1.1.2b20241113
 Summary: Fast and Accurate ML in 3 Lines of Code
 Home-page: https://github.com/autogluon/autogluon
 Author: AutoGluon Community
@@ -56,9 +56,9 @@ Requires-Dist: text-unidecode<1.4,>=1.3
 Requires-Dist: torchmetrics<1.3.0,>=1.2.0
 Requires-Dist: nptyping<2.5.0,>=1.4.4
 Requires-Dist: omegaconf<2.3.0,>=2.1.1
-Requires-Dist: autogluon.core[raytune]==1.1.2b20241111
-Requires-Dist: autogluon.features==1.1.2b20241111
-Requires-Dist: autogluon.common==1.1.2b20241111
+Requires-Dist: autogluon.core[raytune]==1.1.2b20241113
+Requires-Dist: autogluon.features==1.1.2b20241113
+Requires-Dist: autogluon.common==1.1.2b20241113
 Requires-Dist: pytorch-metric-learning<2.4,>=1.3.0
 Requires-Dist: nlpaug<1.2.0,>=1.1.10
 Requires-Dist: nltk<3.9,>=3.4.5

{autogluon.multimodal-1.1.2b20241111.dist-info → autogluon.multimodal-1.1.2b20241113.dist-info}/RECORD

@@ -1,11 +1,11 @@
-autogluon.multimodal-1.1.2b20241111-py3.8-nspkg.pth,sha256=cQGwpuGPqg1GXscIwt-7PmME1OnSpD-7ixkikJ31WAY,554
+autogluon.multimodal-1.1.2b20241113-py3.8-nspkg.pth,sha256=cQGwpuGPqg1GXscIwt-7PmME1OnSpD-7ixkikJ31WAY,554
 autogluon/multimodal/__init__.py,sha256=EuWb-QmtFBKePJw4_4Kpp9dKrabv121haYw_Oiu2jfI,238
-autogluon/multimodal/constants.py,sha256=lFA03ZQeZlp8mwuXLuMOgeByljV5wPYBjN_hi6Xc8zg,8498
+autogluon/multimodal/constants.py,sha256=8IDFqC45Sz3fD0VO2wpzj5Ino387yMAvmKMt-QRhzK0,9122
 autogluon/multimodal/predictor.py,sha256=beV2gOcTnviYtU8UWTWdqWYTbuk5sC6Sba-pAEaFQyg,40936
 autogluon/multimodal/presets.py,sha256=VR_arn7X4eiQcGcvJVmwxDopPJGvYP1W1cBZ2AOcdJM,25882
-autogluon/multimodal/problem_types.py,sha256=JPSoV3Fg-NGQansRlyT2bPZG3iIkgWo7eB8oDoAfg90,9201
+autogluon/multimodal/problem_types.py,sha256=H0q2V--d_KH7YL_AxMrs77SHR5SBVLDsdie9F2Uu1kM,8627
 autogluon/multimodal/registry.py,sha256=vqvONWweZP44aBo1jCvlqLdL0Agn2kyTK8uXUwagZhs,3670
-autogluon/multimodal/version.py,sha256=kh33Q3LDB5QcIlTzUZ64N8Y2DfFFBgbjxMIWQwKPcdk,90
+autogluon/multimodal/version.py,sha256=OGArr6c4pNSqH_iw-M1qDHQMDvnxMN7HervCqm8kQw8,90
 autogluon/multimodal/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autogluon/multimodal/cli/prepare_detection_dataset.py,sha256=9NCYmCUMPRWbxxbN_C7YQjMYlrAm8gbwJ3Qd-79JWH4,5218
 autogluon/multimodal/cli/voc2coco.py,sha256=LXVu9RIfOZs_1URrzO-_3Nrvb9uGEgPxCY4-B6m1coc,9605
@@ -126,7 +126,7 @@ autogluon/multimodal/utils/label_studio.py,sha256=7lFl75zztIy6VCuCbyZkN-BLbtr0j1
 autogluon/multimodal/utils/load.py,sha256=rzfADn6obbZL20QZc034IPhIiza7SA8f5MPpd9hPsAE,5106
 autogluon/multimodal/utils/log.py,sha256=KbVrP5GmQya0e-1YlgdD5EnsUJDGhC7Q3L8eup62KDs,5908
 autogluon/multimodal/utils/matcher.py,sha256=PRtm5LCIXK_L6Qy0WAgMURUYL7M-DTpn9fCFYsCd2OA,18198
-autogluon/multimodal/utils/metric.py,sha256=cnK-NkHmljEvDy5TU4pkHjo32rADlOL7VLA88QSypM4,16286
+autogluon/multimodal/utils/metric.py,sha256=EGlpCwUe5KF9gQ9CPcwDVE5bCgR_oW2r1n4sXdLYXAA,16714
 autogluon/multimodal/utils/misc.py,sha256=WaDWN-6xCCL4tCkxMr4VMb5oiNmmBLrWo5FC3bCQp2A,4772
 autogluon/multimodal/utils/mmcv.py,sha256=Jjg5PiPqiRNJk6yWkQQlNiqT7qhStN94QjqQsZO3uVw,922
 autogluon/multimodal/utils/model.py,sha256=Z_9bev8nRk92cUZjPggVAM3R3CHFlecU-YzjkMGPUsE,21963
@@ -135,11 +135,11 @@ autogluon/multimodal/utils/object_detection.py,sha256=fHZxon5LoYRmNu_7jm_pDjesVx
 autogluon/multimodal/utils/onnx.py,sha256=rblWnphKTsfbosbieJu8PsH6SMDw4on9BS8bR1plL2U,5607
 autogluon/multimodal/utils/save.py,sha256=zYIO3mYMGBvHfZcmCUaLpsQa14nVq1LPv2F76uaz89w,3951
 autogluon/multimodal/utils/visualizer.py,sha256=qAc4_36r3MBDPq1R1jBeb_gcfzIrsylL1S31sRf3wuw,22562
-autogluon.multimodal-1.1.2b20241111.dist-info/LICENSE,sha256=CeipvOyAZxBGUsFoaFqwkx54aPnIKEtm9a5u2uXxEws,10142
-autogluon.multimodal-1.1.2b20241111.dist-info/METADATA,sha256=lf7uI4CLovhQJomHJ1wRR9e2JXKFDR9OFXhzinP92dM,12880
-autogluon.multimodal-1.1.2b20241111.dist-info/NOTICE,sha256=7nPQuj8Kp-uXsU0S5so3-2dNU5EctS5hDXvvzzehd7E,114
-autogluon.multimodal-1.1.2b20241111.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-autogluon.multimodal-1.1.2b20241111.dist-info/namespace_packages.txt,sha256=giERA4R78OkJf2ijn5slgjURlhRPzfLr7waIcGkzYAo,10
-autogluon.multimodal-1.1.2b20241111.dist-info/top_level.txt,sha256=giERA4R78OkJf2ijn5slgjURlhRPzfLr7waIcGkzYAo,10
-autogluon.multimodal-1.1.2b20241111.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-autogluon.multimodal-1.1.2b20241111.dist-info/RECORD,,
+autogluon.multimodal-1.1.2b20241113.dist-info/LICENSE,sha256=CeipvOyAZxBGUsFoaFqwkx54aPnIKEtm9a5u2uXxEws,10142
+autogluon.multimodal-1.1.2b20241113.dist-info/METADATA,sha256=uQzSe85glheqQXVYS_syaHB8NZ-FUn1ZYE018fBa3Jg,12880
+autogluon.multimodal-1.1.2b20241113.dist-info/NOTICE,sha256=7nPQuj8Kp-uXsU0S5so3-2dNU5EctS5hDXvvzzehd7E,114
+autogluon.multimodal-1.1.2b20241113.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
+autogluon.multimodal-1.1.2b20241113.dist-info/namespace_packages.txt,sha256=giERA4R78OkJf2ijn5slgjURlhRPzfLr7waIcGkzYAo,10
+autogluon.multimodal-1.1.2b20241113.dist-info/top_level.txt,sha256=giERA4R78OkJf2ijn5slgjURlhRPzfLr7waIcGkzYAo,10
+autogluon.multimodal-1.1.2b20241113.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+autogluon.multimodal-1.1.2b20241113.dist-info/RECORD,,