valor-lite 0.33.5__py3-none-any.whl → 0.33.7__py3-none-any.whl

valor_lite/classification/metric.py

@@ -0,0 +1,191 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from valor_lite.schemas import Metric
+
+
+class MetricType(Enum):
+    Counts = "Counts"
+    ROCAUC = "ROCAUC"
+    mROCAUC = "mROCAUC"
+    Precision = "Precision"
+    Recall = "Recall"
+    Accuracy = "Accuracy"
+    F1 = "F1"
+    ConfusionMatrix = "ConfusionMatrix"
+
+    @classmethod
+    def base(cls):
+        return [
+            cls.Counts,
+            cls.ROCAUC,
+            cls.mROCAUC,
+            cls.Precision,
+            cls.Recall,
+            cls.Accuracy,
+            cls.F1,
+        ]
+
+
+@dataclass
+class Counts:
+    tp: list[int]
+    fp: list[int]
+    fn: list[int]
+    tn: list[int]
+    score_thresholds: list[float]
+    hardmax: bool
+    label: tuple[str, str]
+
+    @property
+    def metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value={
+                "tp": self.tp,
+                "fp": self.fp,
+                "fn": self.fn,
+                "tn": self.tn,
+            },
+            parameters={
+                "score_thresholds": self.score_thresholds,
+                "hardmax": self.hardmax,
+                "label": {
+                    "key": self.label[0],
+                    "value": self.label[1],
+                },
+            },
+        )
+
+    def to_dict(self) -> dict:
+        return self.metric.to_dict()
+
+
+@dataclass
+class _ThresholdValue:
+    value: list[float]
+    score_thresholds: list[float]
+    hardmax: bool
+    label: tuple[str, str]
+
+    @property
+    def metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value=self.value,
+            parameters={
+                "score_thresholds": self.score_thresholds,
+                "hardmax": self.hardmax,
+                "label": {
+                    "key": self.label[0],
+                    "value": self.label[1],
+                },
+            },
+        )
+
+    def to_dict(self) -> dict:
+        return self.metric.to_dict()
+
+
+class Precision(_ThresholdValue):
+    pass
+
+
+class Recall(_ThresholdValue):
+    pass
+
+
+class Accuracy(_ThresholdValue):
+    pass
+
+
+class F1(_ThresholdValue):
+    pass
+
+
+@dataclass
+class ROCAUC:
+    value: float
+    label: tuple[str, str]
+
+    @property
+    def metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value=self.value,
+            parameters={
+                "label": {
+                    "key": self.label[0],
+                    "value": self.label[1],
+                },
+            },
+        )
+
+    def to_dict(self) -> dict:
+        return self.metric.to_dict()
+
+
+@dataclass
+class mROCAUC:
+    value: float
+    label_key: str
+
+    @property
+    def metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value=self.value,
+            parameters={
+                "label_key": self.label_key,
+            },
+        )
+
+    def to_dict(self) -> dict:
+        return self.metric.to_dict()
+
+
+@dataclass
+class ConfusionMatrix:
+    confusion_matrix: dict[
+        str,  # ground truth label value
+        dict[
+            str,  # prediction label value
+            dict[
+                str,  # either `count` or `examples`
+                int
+                | list[
+                    dict[
+                        str,  # either `datum` or `score`
+                        str | float,  # datum uid  # prediction score
+                    ]
+                ],
+            ],
+        ],
+    ]
+    missing_predictions: dict[
+        str,  # ground truth label value
+        dict[
+            str,  # either `count` or `examples`
+            int | list[dict[str, str]],  # count or datum examples
+        ],
+    ]
+    score_threshold: float
+    label_key: str
+    number_of_examples: int
+
+    @property
+    def metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value={
+                "confusion_matrix": self.confusion_matrix,
+                "missing_predictions": self.missing_predictions,
+            },
+            parameters={
+                "score_threshold": self.score_threshold,
+                "label_key": self.label_key,
+            },
+        )
+
+    def to_dict(self) -> dict:
+        return self.metric.to_dict()

valor_lite/detection/manager.py

@@ -342,6 +342,7 @@ class Evaluator:
         score_thresholds: list[float] = [0.5],
         number_of_examples: int = 0,
         filter_: Filter | None = None,
+        as_dict: bool = False,
     ) -> dict[MetricType, list]:
         """
         Performs an evaluation and returns metrics.
@@ -358,6 +359,8 @@ class Evaluator:
             Maximum number of annotation examples to return in ConfusionMatrix.
         filter_ : Filter, optional
             An optional filter object.
+        as_dict : bool, default=False
+            An option to return metrics as dictionaries.
 
         Returns
         -------
@@ -559,6 +562,12 @@ class Evaluator:
             if metric not in metrics_to_return:
                 del metrics[metric]
 
+        if as_dict:
+            return {
+                mtype: [metric.to_dict() for metric in mvalues]
+                for mtype, mvalues in metrics.items()
+            }
+
         return metrics
 
     def _unpack_confusion_matrix(
@@ -830,22 +839,24 @@ class Evaluator:
         self,
         data: NDArray[np.floating],
         label_metadata: NDArray[np.int32],
-        iou_thresholds: list[float] = [0.5],
-        score_thresholds: list[float] = [
-            score / 10.0 for score in range(1, 11)
-        ],
-        number_of_examples: int = 0,
+        iou_thresholds: list[float],
+        score_thresholds: list[float],
+        number_of_examples: int,
     ) -> list[ConfusionMatrix]:
         """
         Computes detailed counting metrics.
 
         Parameters
         ----------
-        iou_thresholds : list[float], default=[0.5]
+        data : NDArray[np.floating]
+            An array containing detailed pairs of detections.
+        label_metadata : NDArray[np.int32]
+            An array containing label metadata.
+        iou_thresholds : list[float]
             List of IoU thresholds to compute metrics for.
-        score_thresholds : list[float], default=[0.1,0.2,...,1.0]
+        score_thresholds : list[float]
             List of confidence thresholds to compute metrics for.
-        number_of_examples : int, default=0
+        number_of_examples : int
             Maximum number of annotation examples to return per metric.
 
         Returns

{valor_lite-0.33.5.dist-info → valor_lite-0.33.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.5
+Version: 0.33.7
 Summary: Compute valor metrics directly in your client.
 License: MIT License
         

valor_lite-0.33.7.dist-info/RECORD

@@ -0,0 +1,17 @@
+valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
+valor_lite/classification/__init__.py,sha256=2wmmziIzUATm7MbmAcPNLXrEX5l4oeD7XBwPd9bWM3Q,506
+valor_lite/classification/annotation.py,sha256=rMDTvPHdAlvJ6_M2kRrnJQnj1oqKe-lxbncWC7Q50RE,345
+valor_lite/classification/computation.py,sha256=pqAPX6zFlaWyYBnve4sdgJLba_m7smeaqZAsEBvi1no,12776
+valor_lite/classification/manager.py,sha256=Tx6SpEBnV17V-rT6b4MG5jQN-fqG2dlau2-aBnzF_mI,27965
+valor_lite/classification/metric.py,sha256=00qmagf-zQXUZ1qJW_UmN1k35aaYK_7GEM292Tc_cBE,4256
+valor_lite/detection/__init__.py,sha256=PiKfemo8FkZRzBhPSjhil8ahGURLy0Vk_iV25CB4UBU,1139
+valor_lite/detection/annotation.py,sha256=BspLc3SjWXj6qYlGGpzDPHEZ8j7CiFzIL5cNlk0WCAM,2732
+valor_lite/detection/computation.py,sha256=HDFfPTFQN2obm-g570KKDf7SP9V-h09OyMtFEJXsoTA,26323
+valor_lite/detection/manager.py,sha256=BnLqDGaP5h1aC5D_Vm6-oUYFlz-1yuQqlJAnQ1zztSI,53160
+valor_lite/detection/metric.py,sha256=RYKN17nEFRIZIqmotQa6OyNnU0nkjXyfFIclX_5hGpY,9933
+valor_lite-0.33.7.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite-0.33.7.dist-info/METADATA,sha256=EyxuCPqIDbQa4PAQ0utdpb0TVmZUn8TfwSTHnvoKXBc,1865
+valor_lite-0.33.7.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+valor_lite-0.33.7.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.33.7.dist-info/RECORD,,

valor_lite-0.33.5.dist-info/RECORD

@@ -1,12 +0,0 @@
-valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
-valor_lite/detection/__init__.py,sha256=PiKfemo8FkZRzBhPSjhil8ahGURLy0Vk_iV25CB4UBU,1139
-valor_lite/detection/annotation.py,sha256=BspLc3SjWXj6qYlGGpzDPHEZ8j7CiFzIL5cNlk0WCAM,2732
-valor_lite/detection/computation.py,sha256=HDFfPTFQN2obm-g570KKDf7SP9V-h09OyMtFEJXsoTA,26323
-valor_lite/detection/manager.py,sha256=ld2ytAw96UOO25iTwnfvAI1D2UY2Z1AGmP7cyCrT-V4,52801
-valor_lite/detection/metric.py,sha256=RYKN17nEFRIZIqmotQa6OyNnU0nkjXyfFIclX_5hGpY,9933
-valor_lite-0.33.5.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
-valor_lite-0.33.5.dist-info/METADATA,sha256=WL0LQR2fT4CO4MuV0aXIkLPt3zQW2SsBS4MwcA_kHJY,1865
-valor_lite-0.33.5.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-valor_lite-0.33.5.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.33.5.dist-info/RECORD,,