valor-lite 0.33.6__tar.gz → 0.33.8__tar.gz

{valor_lite-0.33.6 → valor_lite-0.33.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.6
+Version: 0.33.8
 Summary: Compute valor metrics directly in your client.
 License: MIT License
         

{valor_lite-0.33.6 → valor_lite-0.33.8}/tests/classification/conftest.py

@@ -7,34 +7,34 @@ def basic_classifications() -> list[Classification]:
     return [
         Classification(
             uid="uid0",
-            groundtruths=[("class", "0")],
+            groundtruth="0",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
-                ("class", "3"),
+                "0",
+                "1",
+                "2",
+                "3",
             ],
             scores=[1.0, 0.0, 0.0, 0.0],
         ),
         Classification(
             uid="uid1",
-            groundtruths=[("class", "0")],
+            groundtruth="0",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
-                ("class", "3"),
+                "0",
+                "1",
+                "2",
+                "3",
             ],
             scores=[0.0, 0.0, 1.0, 0.0],
         ),
         Classification(
             uid="uid2",
-            groundtruths=[("class", "3")],
+            groundtruth="3",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
-                ("class", "3"),
+                "0",
+                "1",
+                "2",
+                "3",
             ],
             scores=[0.0, 0.0, 0.0, 0.3],
         ),
@@ -57,61 +57,61 @@ def classifications_from_api_unit_tests() -> list[Classification]:
     return [
         Classification(
             uid="uid0",
-            groundtruths=[("class", "0")],
+            groundtruth="0",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
+                "0",
+                "1",
+                "2",
             ],
             scores=[1.0, 0.0, 0.0],
         ),
         Classification(
             uid="uid1",
-            groundtruths=[("class", "0")],
+            groundtruth="0",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
+                "0",
+                "1",
+                "2",
             ],
             scores=[0.0, 1.0, 0.0],
         ),
         Classification(
             uid="uid2",
-            groundtruths=[("class", "0")],
+            groundtruth="0",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
+                "0",
+                "1",
+                "2",
             ],
             scores=[0.0, 0.0, 1.0],
         ),
         Classification(
             uid="uid3",
-            groundtruths=[("class", "1")],
+            groundtruth="1",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
+                "0",
+                "1",
+                "2",
             ],
             scores=[0.0, 1.0, 0.0],
         ),
         Classification(
             uid="uid4",
-            groundtruths=[("class", "2")],
+            groundtruth="2",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
+                "0",
+                "1",
+                "2",
             ],
             scores=[0.0, 1.0, 0.0],
         ),
         Classification(
             uid="uid5",
-            groundtruths=[("class", "2")],
+            groundtruth="2",
             predictions=[
-                ("class", "0"),
-                ("class", "1"),
-                ("class", "2"),
+                "0",
+                "1",
+                "2",
             ],
             scores=[0.0, 1.0, 0.0],
         ),
@@ -119,7 +119,7 @@ def classifications_from_api_unit_tests() -> list[Classification]:
 
 
 @pytest.fixture
-def classifications_two_categories() -> list[Classification]:
+def classifications_animal_example() -> list[Classification]:
     animal_gts = ["bird", "dog", "bird", "bird", "cat", "dog"]
     animal_pds = [
         {"bird": 0.6, "dog": 0.2, "cat": 0.2},
@@ -131,6 +131,19 @@ def classifications_two_categories() -> list[Classification]:
         # Note: In the case of a tied score, the ordering of predictions is used.
     ]
 
+    return [
+        Classification(
+            uid=f"uid{idx}",
+            groundtruth=gt,
+            predictions=list(pd.keys()),
+            scores=list(pd.values()),
+        )
+        for idx, (gt, pd) in enumerate(zip(animal_gts, animal_pds))
+    ]
+
+
+@pytest.fixture
+def classifications_color_example() -> list[Classification]:
     color_gts = ["white", "white", "red", "blue", "black", "red"]
     color_pds = [
         {"white": 0.65, "red": 0.1, "blue": 0.2, "black": 0.05},
@@ -141,29 +154,14 @@ def classifications_two_categories() -> list[Classification]:
         {"red": 0.9, "white": 0.06, "blue": 0.01, "black": 0.03},
     ]
 
-    joint_gts = zip(animal_gts, color_gts)
-    joint_pds = [
-        {
-            "animal": animal,
-            "color": color,
-        }
-        for animal, color in zip(animal_pds, color_pds)
-    ]
-
     return [
         Classification(
             uid=f"uid{idx}",
-            groundtruths=[("animal", gt[0]), ("color", gt[1])],
-            predictions=[
-                (key, value)
-                for key, values in pd.items()
-                for value in values.keys()
-            ],
-            scores=[
-                score for values in pd.values() for score in values.values()
-            ],
+            groundtruth=gt,
+            predictions=list(pd.keys()),
+            scores=list(pd.values()),
         )
-        for idx, (gt, pd) in enumerate(zip(joint_gts, joint_pds))
+        for idx, (gt, pd) in enumerate(zip(color_gts, color_pds))
     ]
 
 
@@ -172,36 +170,21 @@ def classifications_image_example() -> list[Classification]:
     return [
         Classification(
             uid="uid5",
-            groundtruths=[
-                ("k4", "v4"),
-                ("k5", "v5"),
-            ],
+            groundtruth="v4",
             predictions=[
-                ("k4", "v1"),
-                ("k4", "v8"),
-                ("k5", "v1"),
+                "v1",
+                "v8",
             ],
-            scores=[0.47, 0.53, 1.0],
+            scores=[0.47, 0.53],
         ),
         Classification(
             uid="uid6",
-            groundtruths=[
-                ("k4", "v4"),
-            ],
-            predictions=[("k4", "v4"), ("k4", "v5")],
-            scores=[0.71, 0.29],
-        ),
-        Classification(
-            uid="uid8",
-            groundtruths=[
-                ("k3", "v3"),
-            ],
+            groundtruth="v4",
             predictions=[
-                ("k3", "v1"),
-            ],
-            scores=[
-                1.0,
+                "v4",
+                "v5",
             ],
+            scores=[0.71, 0.29],
         ),
     ]
 
@@ -224,10 +207,8 @@ def classifications_tabular_example() -> list[Classification]:
     return [
         Classification(
             uid=f"uid{i}",
-            groundtruths=[("class", str(gt_label))],
-            predictions=[
-                ("class", str(pd_label)) for pd_label, _ in enumerate(pds)
-            ],
+            groundtruth=str(gt_label),
+            predictions=[str(pd_label) for pd_label, _ in enumerate(pds)],
             scores=pds,
         )
         for i, (gt_label, pds) in enumerate(
@@ -236,24 +217,12 @@ def classifications_tabular_example() -> list[Classification]:
     ]
 
 
-@pytest.fixture
-def classifications_no_groundtruths() -> list[Classification]:
-    return [
-        Classification(
-            uid="uid1",
-            groundtruths=[],
-            predictions=[("k1", "v1"), ("k1", "v2")],
-            scores=[0.8, 0.2],
-        )
-    ]
-
-
 @pytest.fixture
 def classifications_no_predictions() -> list[Classification]:
     return [
         Classification(
             uid="uid1",
-            groundtruths=[("k1", "v1"), ("k2", "v2")],
+            groundtruth="v1",
             predictions=[],
             scores=[],
         )
@@ -265,11 +234,11 @@ def classifications_multiclass() -> list[Classification]:
     return [
         Classification(
             uid="uid0",
-            groundtruths=[("class_label", "cat")],
+            groundtruth="cat",
             predictions=[
-                ("class_label", "cat"),
-                ("class_label", "dog"),
-                ("class_label", "bee"),
+                "cat",
+                "dog",
+                "bee",
             ],
             scores=[
                 0.44598543489942505,
@@ -279,11 +248,11 @@ def classifications_multiclass() -> list[Classification]:
         ),
         Classification(
             uid="uid1",
-            groundtruths=[("class_label", "bee")],
+            groundtruth="bee",
             predictions=[
-                ("class_label", "cat"),
-                ("class_label", "dog"),
-                ("class_label", "bee"),
+                "cat",
+                "dog",
+                "bee",
             ],
             scores=[
                 0.4076893257212283,
@@ -293,11 +262,11 @@ def classifications_multiclass() -> list[Classification]:
         ),
         Classification(
             uid="uid2",
-            groundtruths=[("class_label", "cat")],
+            groundtruth="cat",
             predictions=[
-                ("class_label", "cat"),
-                ("class_label", "dog"),
-                ("class_label", "bee"),
+                "cat",
+                "dog",
+                "bee",
             ],
             scores=[
                 0.25060075263871917,
@@ -307,11 +276,11 @@ def classifications_multiclass() -> list[Classification]:
         ),
         Classification(
             uid="uid3",
-            groundtruths=[("class_label", "bee")],
+            groundtruth="bee",
             predictions=[
-                ("class_label", "cat"),
-                ("class_label", "dog"),
-                ("class_label", "bee"),
+                "cat",
+                "dog",
+                "bee",
             ],
             scores=[
                 0.2003514145616792,
@@ -321,11 +290,11 @@ def classifications_multiclass() -> list[Classification]:
         ),
         Classification(
             uid="uid4",
-            groundtruths=[("class_label", "dog")],
+            groundtruth="dog",
             predictions=[
-                ("class_label", "cat"),
-                ("class_label", "dog"),
-                ("class_label", "bee"),
+                "cat",
+                "dog",
+                "bee",
             ],
             scores=[
                 0.33443897813714385,
@@ -337,18 +306,14 @@ def classifications_multiclass() -> list[Classification]:
 
 
 @pytest.fixture
-def classifications_multiclass_true_negatives_check() -> list[Classification]:
+def classifications_multiclass_true_negatives_check() -> (
+    list[Classification]
+):
     return [
         Classification(
             uid="uid1",
-            groundtruths=[("k1", "ant")],
-            predictions=[("k1", "ant"), ("k1", "bee"), ("k1", "cat")],
-            scores=[0.15, 0.48, 0.37],
-        ),
-        Classification(
-            uid="uid2",
-            groundtruths=[("k2", "egg")],
-            predictions=[("k2", "egg"), ("k2", "milk"), ("k2", "flour")],
+            groundtruth="ant",
+            predictions=["ant", "bee", "cat"],
             scores=[0.15, 0.48, 0.37],
         ),
     ]
@@ -359,20 +324,8 @@ def classifications_multiclass_zero_count() -> list[Classification]:
     return [
         Classification(
             uid="uid1",
-            groundtruths=[("k", "ant")],
-            predictions=[("k", "ant"), ("k", "bee"), ("k", "cat")],
+            groundtruth="ant",
+            predictions=["ant", "bee", "cat"],
             scores=[0.15, 0.48, 0.37],
         )
     ]
-
-
-@pytest.fixture
-def classifications_with_label_key_mismatch() -> list[Classification]:
-    return [
-        Classification(
-            uid="uid0",
-            groundtruths=[("k1", "V1")],
-            predictions=[("k2", "v1")],
-            scores=[1.0],
-        )
-    ]

{valor_lite-0.33.6 → valor_lite-0.33.8}/tests/classification/test_accuracy.py

@@ -77,16 +77,12 @@ def test_accuracy_basic(basic_classifications: list[Classification]):
         "n_groundtruths": 3,
         "n_predictions": 12,
         "n_labels": 4,
-        "ignored_prediction_labels": [
-            ("class", "1"),
-            ("class", "2"),
-        ],
+        "ignored_prediction_labels": ["1", "2"],
         "missing_prediction_labels": [],
     }
 
     metrics = evaluator.evaluate(score_thresholds=[0.25, 0.75], as_dict=True)
 
-    # test Accuracy
     actual_metrics = [m for m in metrics[MetricType.Accuracy]]
     expected_metrics = [
         {
@@ -95,7 +91,7 @@ def test_accuracy_basic(basic_classifications: list[Classification]):
             "parameters": {
                 "score_thresholds": [0.25, 0.75],
                 "hardmax": True,
-                "label": {"key": "class", "value": "0"},
+                "label": "0",
             },
         },
         {
@@ -104,7 +100,7 @@ def test_accuracy_basic(basic_classifications: list[Classification]):
             "parameters": {
                 "score_thresholds": [0.25, 0.75],
                 "hardmax": True,
-                "label": {"key": "class", "value": "3"},
+                "label": "3",
             },
         },
     ]
@@ -114,17 +110,16 @@ def test_accuracy_basic(basic_classifications: list[Classification]):
         assert m in actual_metrics
 
 
-def test_accuracy_with_example(
-    classifications_two_categories: list[Classification],
+def test_accuracy_with_animal_example(
+    classifications_animal_example: list[Classification],
 ):
 
     loader = DataLoader()
-    loader.add_data(classifications_two_categories)
+    loader.add_data(classifications_animal_example)
     evaluator = loader.finalize()
 
     metrics = evaluator.evaluate(score_thresholds=[0.5], as_dict=True)
 
-    # test Accuracy
     actual_metrics = [m for m in metrics[MetricType.Accuracy]]
     expected_metrics = [
         {
@@ -133,7 +128,7 @@ def test_accuracy_with_example(
             "parameters": {
                 "score_thresholds": [0.5],
                 "hardmax": True,
-                "label": {"key": "animal", "value": "bird"},
+                "label": "bird",
             },
         },
         {
@@ -142,7 +137,7 @@ def test_accuracy_with_example(
             "parameters": {
                 "score_thresholds": [0.5],
                 "hardmax": True,
-                "label": {"key": "animal", "value": "dog"},
+                "label": "dog",
             },
         },
         {
@@ -151,16 +146,35 @@ def test_accuracy_with_example(
             "parameters": {
                 "score_thresholds": [0.5],
                 "hardmax": True,
-                "label": {"key": "animal", "value": "cat"},
+                "label": "cat",
             },
         },
+    ]
+    for m in actual_metrics:
+        assert m in expected_metrics
+    for m in expected_metrics:
+        assert m in actual_metrics
+
+
+def test_accuracy_color_example(
+    classifications_color_example: list[Classification],
+):
+
+    loader = DataLoader()
+    loader.add_data(classifications_color_example)
+    evaluator = loader.finalize()
+
+    metrics = evaluator.evaluate(score_thresholds=[0.5], as_dict=True)
+
+    actual_metrics = [m for m in metrics[MetricType.Accuracy]]
+    expected_metrics = [
         {
             "type": "Accuracy",
             "value": [2 / 3],
             "parameters": {
                 "score_thresholds": [0.5],
                 "hardmax": True,
-                "label": {"key": "color", "value": "white"},
+                "label": "white",
             },
         },
         {
@@ -169,7 +183,7 @@ def test_accuracy_with_example(
             "parameters": {
                 "score_thresholds": [0.5],
                 "hardmax": True,
-                "label": {"key": "color", "value": "red"},
+                "label": "red",
             },
         },
         {
@@ -178,7 +192,7 @@ def test_accuracy_with_example(
             "parameters": {
                 "score_thresholds": [0.5],
                 "hardmax": True,
-                "label": {"key": "color", "value": "blue"},
+                "label": "blue",
             },
         },
         {
@@ -187,7 +201,7 @@ def test_accuracy_with_example(
             "parameters": {
                 "score_thresholds": [0.5],
                 "hardmax": True,
-                "label": {"key": "color", "value": "black"},
+                "label": "black",
             },
         },
     ]
@@ -205,53 +219,25 @@ def test_accuracy_with_image_example(
     evaluator = loader.finalize()
 
     assert evaluator.metadata == {
-        "n_datums": 3,
-        "n_groundtruths": 4,
-        "n_predictions": 6,
-        "n_labels": 8,
-        "ignored_prediction_labels": [
-            ("k4", "v1"),
-            ("k4", "v8"),
-            ("k5", "v1"),
-            ("k4", "v5"),
-            ("k3", "v1"),
-        ],
-        "missing_prediction_labels": [
-            ("k5", "v5"),
-            ("k3", "v3"),
-        ],
+        "n_datums": 2,
+        "n_groundtruths": 2,
+        "n_predictions": 4,
+        "n_labels": 4,
+        "ignored_prediction_labels": ["v1", "v8", "v5"],
+        "missing_prediction_labels": [],
     }
 
     metrics = evaluator.evaluate(as_dict=True)
 
-    # test Accuracy
     actual_metrics = [m for m in metrics[MetricType.Accuracy]]
     expected_metrics = [
         {
             "type": "Accuracy",
-            "value": [0.3333333333333333],
-            "parameters": {
-                "score_thresholds": [0.0],
-                "hardmax": True,
-                "label": {"key": "k4", "value": "v4"},
-            },
-        },
-        {
-            "type": "Accuracy",
-            "value": [0.0],
-            "parameters": {
-                "score_thresholds": [0.0],
-                "hardmax": True,
-                "label": {"key": "k5", "value": "v5"},
-            },
-        },
-        {
-            "type": "Accuracy",
-            "value": [0.0],
+            "value": [0.5],
             "parameters": {
                 "score_thresholds": [0.0],
                 "hardmax": True,
-                "label": {"key": "k3", "value": "v3"},
+                "label": "v4",
             },
         },
     ]
@@ -279,7 +265,6 @@ def test_accuracy_with_tabular_example(
 
     metrics = evaluator.evaluate(as_dict=True)
 
-    # test Accuracy
     actual_metrics = [m for m in metrics[MetricType.Accuracy]]
     expected_metrics = [
         {
@@ -288,7 +273,7 @@ def test_accuracy_with_tabular_example(
             "parameters": {
                 "score_thresholds": [0.0],
                 "hardmax": True,
-                "label": {"key": "class", "value": "0"},
+                "label": "0",
             },
         },
         {
@@ -297,7 +282,7 @@ def test_accuracy_with_tabular_example(
             "parameters": {
                 "score_thresholds": [0.0],
                 "hardmax": True,
-                "label": {"key": "class", "value": "1"},
+                "label": "1",
             },
         },
         {
@@ -306,7 +291,7 @@ def test_accuracy_with_tabular_example(
             "parameters": {
                 "score_thresholds": [0.0],
                 "hardmax": True,
-                "label": {"key": "class", "value": "2"},
+                "label": "2",
             },
         },
     ]