autogluon.multimodal 1.1.2b20241111__tar.gz → 1.1.2b20241113__tar.gz

{autogluon.multimodal-1.1.2b20241111 → autogluon.multimodal-1.1.2b20241113}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: autogluon.multimodal
-Version: 1.1.2b20241111
+Version: 1.1.2b20241113
 Summary: Fast and Accurate ML in 3 Lines of Code
 Home-page: https://github.com/autogluon/autogluon
 Author: AutoGluon Community

{autogluon.multimodal-1.1.2b20241111 → autogluon.multimodal-1.1.2b20241113}/src/autogluon/multimodal/constants.py

@@ -177,6 +177,23 @@ MATCHING_METRICS = {
 }
 MATCHING_METRICS_WITHOUT_PROBLEM_TYPE = [RECALL, NDCG]
 
+EVALUATION_METRICS = {
+    # Use evaluation metrics from METRICS for these types
+    BINARY: METRICS[BINARY].keys(),
+    MULTICLASS: METRICS[MULTICLASS].keys(),
+    REGRESSION: METRICS[REGRESSION].keys(),
+    OBJECT_DETECTION: DETECTION_METRICS,
+    SEMANTIC_SEGMENTATION: [IOU, BER, SM],
+    NER: [OVERALL_F1, NER_TOKEN_F1],
+    NAMED_ENTITY_RECOGNITION: [OVERALL_F1, NER_TOKEN_F1],
+    FEW_SHOT_CLASSIFICATION: METRICS[MULTICLASS].keys(),
+}
+
+VALIDATION_METRICS = {
+    problem_type: [metric for metric in metrics if metric in METRIC_MODE_MAP] + [DIRECT_LOSS]
+    for problem_type, metrics in EVALUATION_METRICS.items()
+}
+
 # Training status
 TRAIN = "train"
 VALIDATE = "validate"

{autogluon.multimodal-1.1.2b20241111 → autogluon.multimodal-1.1.2b20241113}/src/autogluon/multimodal/problem_types.py

@@ -3,16 +3,12 @@
 from dataclasses import dataclass, field
 from typing import List, Optional, Set
 
-from autogluon.core.metrics import METRICS
-
 from .constants import (
     ACCURACY,
-    BER,
     BINARY,
     CATEGORICAL,
     CLASSIFICATION,
-    DETECTION_METRICS,
-    DIRECT_LOSS,
+    EVALUATION_METRICS,
     FEATURE_EXTRACTION,
     FEW_SHOT_CLASSIFICATION,
     IMAGE,
@@ -22,7 +18,6 @@ from .constants import (
     IMAGE_TEXT_SIMILARITY,
     IOU,
     MAP,
-    METRIC_MODE_MAP,
     MULTICLASS,
     NAMED_ENTITY_RECOGNITION,
     NER,
@@ -30,16 +25,15 @@ from .constants import (
     NER_TOKEN_F1,
     NUMERICAL,
     OBJECT_DETECTION,
-    OVERALL_F1,
     REGRESSION,
     RMSE,
     ROC_AUC,
     ROIS,
     SEMANTIC_SEGMENTATION,
-    SM,
     TEXT,
     TEXT_NER,
     TEXT_SIMILARITY,
+    VALIDATION_METRICS,
     ZERO_SHOT_IMAGE_CLASSIFICATION,
 )
 from .registry import Registry
@@ -76,11 +70,10 @@ class ProblemTypeProperty:
     # The fallback label type of the problem
     _fallback_label_type: Optional[str] = None
 
-    # The evaluation metrics supported
-    # The FIRST metric is used as fallback by default
-    _supported_evaluation_metrics: Optional[List[str]] = None
+    # If evaluations are supported
+    _support_eval: Optional[bool] = False
 
-    # Overwrite the default setting (first in _supported_evaluation_metrics)
+    # Overwrite the default setting (first in supported_evaluation_metrics)
     _fallback_evaluation_metric: Optional[str] = None
 
     # The validation metric fallback
@@ -96,8 +89,8 @@ class ProblemTypeProperty:
 
     @property
     def supported_evaluation_metrics(self):
-        if self._supported_evaluation_metrics:
-            return self._supported_evaluation_metrics
+        if self._support_eval:
+            return EVALUATION_METRICS[self.name]
         else:
             return []
 
@@ -105,17 +98,15 @@ class ProblemTypeProperty:
     def fallback_evaluation_metric(self):
         if self._fallback_evaluation_metric:
             return self._fallback_evaluation_metric
-        elif self._supported_evaluation_metrics:
-            return self._supported_evaluation_metrics[0]
+        elif self._support_eval:
+            return self.supported_evaluation_metrics[0]
         else:
             return None
 
     @property
     def supported_validation_metrics(self):
-        if self._supported_evaluation_metrics:
-            return [metric for metric in self._supported_evaluation_metrics if metric in METRIC_MODE_MAP.keys()] + [
-                DIRECT_LOSS
-            ]
+        if self._support_eval:
+            return VALIDATION_METRICS[self.name]
         else:
             return []
 
@@ -145,7 +136,7 @@ PROBLEM_TYPES_REG.register(
         supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},
         supported_label_type={CATEGORICAL},
         is_classification=True,
-        _supported_evaluation_metrics=METRICS[BINARY].keys(),
+        _support_eval=True,
         _fallback_evaluation_metric=ROC_AUC,
         _fallback_validation_metric=ROC_AUC,
     ),
@@ -157,7 +148,7 @@ PROBLEM_TYPES_REG.register(
         supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},
         supported_label_type={CATEGORICAL},
         is_classification=True,
-        _supported_evaluation_metrics=METRICS[MULTICLASS].keys(),
+        _support_eval=True,
         _fallback_evaluation_metric=ACCURACY,
         _fallback_validation_metric=ACCURACY,
     ),
@@ -170,7 +161,7 @@ PROBLEM_TYPES_REG.register(
         name=REGRESSION,
         supported_modality_type={IMAGE, IMAGE_BYTEARRAY, IMAGE_BASE64_STR, TEXT, CATEGORICAL, NUMERICAL},
         supported_label_type={NUMERICAL},
-        _supported_evaluation_metrics=METRICS[REGRESSION].keys(),
+        _support_eval=True,
         _fallback_evaluation_metric=RMSE,
         _fallback_validation_metric=RMSE,
     ),
@@ -185,7 +176,7 @@ PROBLEM_TYPES_REG.register(
         supported_modality_type={IMAGE},
         supported_label_type={ROIS},
         force_exist_modality={IMAGE},
-        _supported_evaluation_metrics=DETECTION_METRICS,
+        _support_eval=True,
         _fallback_validation_metric=MAP,
     ),
 )
@@ -200,7 +191,7 @@ PROBLEM_TYPES_REG.register(
         supported_modality_type={IMAGE},
         supported_label_type={IMAGE},
         force_exist_modality={IMAGE},
-        _supported_evaluation_metrics=[IOU, BER, SM],
+        _support_eval=True,
         _fallback_evaluation_metric=IOU,
         _fallback_validation_metric=IOU,
     ),
@@ -247,7 +238,7 @@ _ner_property = ProblemTypeProperty(
     supported_modality_type={IMAGE, TEXT, CATEGORICAL, NUMERICAL, TEXT_NER},
     supported_label_type={NER_ANNOTATION},
     force_exist_modality={TEXT_NER},
-    _supported_evaluation_metrics=[OVERALL_F1, NER_TOKEN_F1],
+    _support_eval=True,
     _fallback_validation_metric=NER_TOKEN_F1,
 )
 PROBLEM_TYPES_REG.register(NER, _ner_property)
@@ -281,7 +272,7 @@ PROBLEM_TYPES_REG.register(
         support_zero_shot=False,
         supported_modality_type={IMAGE, TEXT},
         supported_label_type={CATEGORICAL},
-        _supported_evaluation_metrics=METRICS[MULTICLASS].keys(),
+        _support_eval=True,
         _fallback_evaluation_metric=ACCURACY,
         _fallback_validation_metric=ACCURACY,
     ),

{autogluon.multimodal-1.1.2b20241111 → autogluon.multimodal-1.1.2b20241113}/src/autogluon/multimodal/utils/metric.py

@@ -8,7 +8,7 @@ import evaluate
 import numpy as np
 from sklearn.metrics import f1_score
 
-from autogluon.core.metrics import Scorer, get_metric
+from autogluon.core.metrics import Scorer, compute_metric, get_metric
 
 from ..constants import (
     ACCURACY,
@@ -231,15 +231,22 @@ def compute_score(
         return metric.compute(references=metric_data[Y_TRUE], predictions=metric_data[Y_PRED])
 
     metric = get_metric(metric)
-    if metric.name in [ROC_AUC, AVERAGE_PRECISION]:
-        return metric._sign * metric(metric_data[Y_TRUE], metric_data[Y_PRED_PROB][:, pos_label])
-    elif metric.name in [F1]:  # only for binary classification
-        return f1_score(metric_data[Y_TRUE], metric_data[Y_PRED], pos_label=pos_label)
+
+    y = metric_data[Y_TRUE]
+    if metric.needs_proba or metric.needs_threshold:
+        return metric.convert_score_to_original(
+            compute_metric(y=y, y_pred_proba=metric_data[Y_PRED_PROB][:, pos_label], metric=metric, weights=None)
+        )
     else:
-        try:
-            return metric._sign * metric(metric_data[Y_TRUE], metric_data[Y_PRED], y_prob=metric_data[Y_PRED_PROB])
-        except:
-            return metric._sign * metric(metric_data[Y_TRUE], metric_data[Y_PRED])
+        y_pred = metric_data[Y_PRED]
+
+        # TODO: This is a hack. Doesn't support `f1_macro`, `f1_micro`, `f1_weighted`, or custom `f1` metrics with different names.
+        # TODO: Longterm the solution should be to have the input data to this function use the internal representation without the original class names. This way `pos_label` would not need to be specified.
+        if metric.name == F1:  # only for binary classification
+            y = (metric_data[Y_TRUE] == pos_label).astype(int)
+            y_pred = (metric_data[Y_PRED] == pos_label).astype(int)
+
+        return metric.convert_score_to_original(compute_metric(y=y, y_pred=y_pred, metric=metric, weights=None))
 
 
 class RankingMetrics:

{autogluon.multimodal-1.1.2b20241111 → autogluon.multimodal-1.1.2b20241113}/src/autogluon/multimodal/version.py

@@ -1,3 +1,3 @@
 """This is the autogluon version file."""
-__version__ = '1.1.2b20241111'
+__version__ = '1.1.2b20241113'
 __lite__ = False

{autogluon.multimodal-1.1.2b20241111 → autogluon.multimodal-1.1.2b20241113}/src/autogluon.multimodal.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: autogluon.multimodal
-Version: 1.1.2b20241111
+Version: 1.1.2b20241113
 Summary: Fast and Accurate ML in 3 Lines of Code
 Home-page: https://github.com/autogluon/autogluon
 Author: AutoGluon Community

{autogluon.multimodal-1.1.2b20241111 → autogluon.multimodal-1.1.2b20241113}/src/autogluon.multimodal.egg-info/requires.txt

@@ -20,9 +20,9 @@ text-unidecode<1.4,>=1.3
 torchmetrics<1.3.0,>=1.2.0
 nptyping<2.5.0,>=1.4.4
 omegaconf<2.3.0,>=2.1.1
-autogluon.core[raytune]==1.1.2b20241111
-autogluon.features==1.1.2b20241111
-autogluon.common==1.1.2b20241111
+autogluon.core[raytune]==1.1.2b20241113
+autogluon.features==1.1.2b20241113
+autogluon.common==1.1.2b20241113
 pytorch-metric-learning<2.4,>=1.3.0
 nlpaug<1.2.0,>=1.1.10
 nltk<3.9,>=3.4.5