valor-lite 0.33.7__py3-none-any.whl → 0.33.9__py3-none-any.whl

valor_lite/classification/metric.py

@@ -14,31 +14,49 @@ class MetricType(Enum):
     F1 = "F1"
     ConfusionMatrix = "ConfusionMatrix"
 
-    @classmethod
-    def base(cls):
-        return [
-            cls.Counts,
-            cls.ROCAUC,
-            cls.mROCAUC,
-            cls.Precision,
-            cls.Recall,
-            cls.Accuracy,
-            cls.F1,
-        ]
-
 
 @dataclass
 class Counts:
+    """
+    Confusion matrix counts at specified score thresholds for binary classification.
+
+    This class stores the true positive (`tp`), false positive (`fp`), false negative (`fn`), and true
+    negative (`tn`) counts computed at various score thresholds for a binary classification task.
+
+    Attributes
+    ----------
+    tp : list[int]
+        True positive counts at each score threshold.
+    fp : list[int]
+        False positive counts at each score threshold.
+    fn : list[int]
+        False negative counts at each score threshold.
+    tn : list[int]
+        True negative counts at each score threshold.
+    score_thresholds : list[float]
+        Score thresholds at which the counts are computed.
+    hardmax : bool
+        Indicates whether hardmax thresholding was used.
+    label : str
+        The class label for which the counts are computed.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
     tp: list[int]
     fp: list[int]
     fn: list[int]
     tn: list[int]
     score_thresholds: list[float]
     hardmax: bool
-    label: tuple[str, str]
+    label: str
 
-    @property
-    def metric(self) -> Metric:
+    def to_metric(self) -> Metric:
         return Metric(
             type=type(self).__name__,
             value={
@@ -50,15 +68,12 @@ class Counts:
             parameters={
                 "score_thresholds": self.score_thresholds,
                 "hardmax": self.hardmax,
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
+                "label": self.label,
             },
         )
 
     def to_dict(self) -> dict:
-        return self.metric.to_dict()
+        return self.to_metric().to_dict()
 
 
 @dataclass
@@ -66,86 +81,297 @@ class _ThresholdValue:
     value: list[float]
     score_thresholds: list[float]
     hardmax: bool
-    label: tuple[str, str]
+    label: str
 
-    @property
-    def metric(self) -> Metric:
+    def to_metric(self) -> Metric:
         return Metric(
             type=type(self).__name__,
             value=self.value,
             parameters={
                 "score_thresholds": self.score_thresholds,
                 "hardmax": self.hardmax,
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
+                "label": self.label,
             },
         )
 
     def to_dict(self) -> dict:
-        return self.metric.to_dict()
+        return self.to_metric().to_dict()
 
 
 class Precision(_ThresholdValue):
+    """
+    Precision metric for a specific class label.
+
+    This class calculates the precision at various score thresholds for a binary
+    classification task. Precision is defined as the ratio of true positives to the
+    sum of true positives and false positives.
+
+    Attributes
+    ----------
+    value : list[float]
+        Precision values computed at each score threshold.
+    score_thresholds : list[float]
+        Score thresholds at which the precision values are computed.
+    hardmax : bool
+        Indicates whether hardmax thresholding was used.
+    label : str
+        The class label for which the precision is computed.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
     pass
 
 
 class Recall(_ThresholdValue):
+    """
+    Recall metric for a specific class label.
+
+    This class calculates the recall at various score thresholds for a binary
+    classification task. Recall is defined as the ratio of true positives to the
+    sum of true positives and false negatives.
+
+    Attributes
+    ----------
+    value : list[float]
+        Recall values computed at each score threshold.
+    score_thresholds : list[float]
+        Score thresholds at which the recall values are computed.
+    hardmax : bool
+        Indicates whether hardmax thresholding was used.
+    label : str
+        The class label for which the recall is computed.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
     pass
 
 
 class Accuracy(_ThresholdValue):
+    """
+    Accuracy metric for a specific class label.
+
+    This class calculates the accuracy at various score thresholds for a binary
+    classification task. Accuracy is defined as the ratio of the sum of true positives and
+    true negatives over all predictions.
+
+    Attributes
+    ----------
+    value : list[float]
+        Accuracy values computed at each score threshold.
+    score_thresholds : list[float]
+        Score thresholds at which the accuracy values are computed.
+    hardmax : bool
+        Indicates whether hardmax thresholding was used.
+    label : str
+        The class label for which the accuracy is computed.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
     pass
 
 
 class F1(_ThresholdValue):
+    """
+    F1 score for a specific class label.
+
+    This class calculates the F1 score at various score thresholds for a binary
+    classification task.
+
+    Attributes
+    ----------
+    value : list[float]
+        F1 scores computed at each score threshold.
+    score_thresholds : list[float]
+        Score thresholds at which the F1 scores are computed.
+    hardmax : bool
+        Indicates whether hardmax thresholding was used.
+    label : str
+        The class label for which the F1 score is computed.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
     pass
 
 
 @dataclass
 class ROCAUC:
+    """
+    Receiver Operating Characteristic Area Under the Curve (ROC AUC).
+
+    This class calculates the ROC AUC score for a specific class label in a multiclass classification task.
+    ROC AUC is a performance measurement for classification problems at various threshold settings.
+    It reflects the ability of the classifier to distinguish between the positive and negative classes.
+
+    Parameters
+    ----------
+    value : float
+        The computed ROC AUC score.
+    label : str
+        The class label for which the ROC AUC is computed.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
     value: float
-    label: tuple[str, str]
+    label: str
 
-    @property
-    def metric(self) -> Metric:
+    def to_metric(self) -> Metric:
         return Metric(
             type=type(self).__name__,
             value=self.value,
-            parameters={
-                "label": {
-                    "key": self.label[0],
-                    "value": self.label[1],
-                },
-            },
+            parameters={"label": self.label},
         )
 
     def to_dict(self) -> dict:
-        return self.metric.to_dict()
+        return self.to_metric().to_dict()
 
 
 @dataclass
 class mROCAUC:
+    """
+    Mean Receiver Operating Characteristic Area Under the Curve (mROC AUC).
+
+    This class calculates the mean ROC AUC score over all classes in a multiclass classification task.
+    It provides an aggregate measure of the model's ability to distinguish between classes.
+
+    Parameters
+    ----------
+    value : float
+        The computed mean ROC AUC score.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
     value: float
-    label_key: str
 
-    @property
-    def metric(self) -> Metric:
+    def to_metric(self) -> Metric:
         return Metric(
             type=type(self).__name__,
             value=self.value,
-            parameters={
-                "label_key": self.label_key,
-            },
+            parameters={},
         )
 
     def to_dict(self) -> dict:
-        return self.metric.to_dict()
+        return self.to_metric().to_dict()
 
 
 @dataclass
 class ConfusionMatrix:
+    """
+    The confusion matrix and related metrics for the classification task.
+
+    This class encapsulates detailed information about the model's performance, including correct
+    predictions, misclassifications, hallucinations (false positives), and missing predictions
+    (false negatives). It provides counts and examples for each category to facilitate in-depth analysis.
+
+    Confusion Matrix Structure:
+    {
+        ground_truth_label: {
+            predicted_label: {
+                'count': int,
+                'examples': [
+                    {
+                        'datum': str,
+                        'groundtruth': dict,  # {'xmin': float, 'xmax': float, 'ymin': float, 'ymax': float}
+                        'prediction': dict,   # {'xmin': float, 'xmax': float, 'ymin': float, 'ymax': float}
+                        'score': float,
+                    },
+                    ...
+                ],
+            },
+            ...
+        },
+        ...
+    }
+
+    Hallucinations Structure:
+    {
+        prediction_label: {
+            'count': int,
+            'examples': [
+                {
+                    'datum': str,
+                    'prediction': dict,  # {'xmin': float, 'xmax': float, 'ymin': float, 'ymax': float}
+                    'score': float,
+                },
+                ...
+            ],
+        },
+        ...
+    }
+
+    Missing Prediction Structure:
+    {
+        ground_truth_label: {
+            'count': int,
+            'examples': [
+                {
+                    'datum': str,
+                    'groundtruth': dict,  # {'xmin': float, 'xmax': float, 'ymin': float, 'ymax': float}
+                },
+                ...
+            ],
+        },
+        ...
+    }
+
+    Attributes
+    ----------
+    confusion_matrix : dict
+        A nested dictionary where the first key is the ground truth label value, the second key
+        is the prediction label value, and the innermost dictionary contains either a `count`
+        or a list of `examples`. Each example includes the datum UID and prediction score.
+    missing_predictions : dict
+        A dictionary where each key is a ground truth label value for which the model failed to predict
+        (false negatives). The value is a dictionary containing either a `count` or a list of `examples`.
+        Each example includes the datum UID.
+    score_threshold : float
+        The confidence score threshold used to filter predictions.
+    number_of_examples : int
+        The maximum number of examples per element.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
     confusion_matrix: dict[
         str,  # ground truth label value
         dict[
@@ -170,11 +396,9 @@ class ConfusionMatrix:
         ],
     ]
     score_threshold: float
-    label_key: str
     number_of_examples: int
 
-    @property
-    def metric(self) -> Metric:
+    def to_metric(self) -> Metric:
         return Metric(
             type=type(self).__name__,
             value={
@@ -183,9 +407,8 @@ class ConfusionMatrix:
             },
             parameters={
                 "score_threshold": self.score_threshold,
-                "label_key": self.label_key,
             },
         )
 
     def to_dict(self) -> dict:
-        return self.metric.to_dict()
+        return self.to_metric().to_dict()

valor_lite/object_detection/annotation.py

@@ -0,0 +1,274 @@
+from dataclasses import dataclass, field
+
+import numpy as np
+from numpy.typing import NDArray
+from shapely.geometry import Polygon as ShapelyPolygon
+
+
+@dataclass
+class BoundingBox:
+    """
+    Represents a bounding box with associated labels and optional scores.
+
+    Parameters
+    ----------
+    xmin : float
+        The minimum x-coordinate of the bounding box.
+    xmax : float
+        The maximum x-coordinate of the bounding box.
+    ymin : float
+        The minimum y-coordinate of the bounding box.
+    ymax : float
+        The maximum y-coordinate of the bounding box.
+    labels : list of str
+        List of labels associated with the bounding box.
+    scores : list of float, optional
+        Confidence scores corresponding to each label. Defaults to an empty list.
+
+    Examples
+    --------
+    Ground Truth Example:
+
+    >>> bbox = BoundingBox(xmin=10.0, xmax=50.0, ymin=20.0, ymax=60.0, labels=['cat'])
+
+    Prediction Example:
+
+    >>> bbox = BoundingBox(
+    ...     xmin=10.0, xmax=50.0, ymin=20.0, ymax=60.0,
+    ...     labels=['cat', 'dog'], scores=[0.9, 0.1]
+    ... )
+    """
+
+    xmin: float
+    xmax: float
+    ymin: float
+    ymax: float
+    labels: list[str]
+    scores: list[float] = field(default_factory=list)
+
+    def __post_init__(self):
+        if len(self.scores) == 0 and len(self.labels) != 1:
+            raise ValueError(
+                "Ground truths must be defined with no scores and a single label. If you meant to define a prediction, then please include one score for every label provided."
+            )
+        if len(self.scores) > 0 and len(self.labels) != len(self.scores):
+            raise ValueError(
+                "If scores are defined, there must be a 1:1 pairing with labels."
+            )
+
+    @property
+    def extrema(self) -> tuple[float, float, float, float]:
+        """
+        Returns the bounding box extrema.
+
+        Returns
+        -------
+        tuple[float, float, float, float]
+            A tuple in the form (xmin, xmax, ymin, ymax).
+        """
+        return (self.xmin, self.xmax, self.ymin, self.ymax)
+
+    @property
+    def annotation(self) -> tuple[float, float, float, float]:
+        """
+        Returns the annotation's data representation.
+
+        Returns
+        -------
+        tuple[float, float, float, float]
+            A tuple in the form (xmin, xmax, ymin, ymax).
+        """
+        return self.extrema
+
+
+@dataclass
+class Polygon:
+    """
+    Represents a polygon shape with associated labels and optional scores.
+
+    Parameters
+    ----------
+    shape : ShapelyPolygon
+        A Shapely polygon object representing the shape.
+    labels : list of str
+        List of labels associated with the polygon.
+    scores : list of float, optional
+        Confidence scores corresponding to each label. Defaults to an empty list.
+
+    Examples
+    --------
+    Ground Truth Example:
+
+    >>> from shapely.geometry import Polygon as ShapelyPolygon
+    >>> shape = ShapelyPolygon([(0, 0), (1, 0), (1, 1), (0, 1)])
+    >>> polygon = Polygon(shape=shape, labels=['building'])
+
+    Prediction Example:
+
+    >>> polygon = Polygon(
+    ...     shape=shape, labels=['building'], scores=[0.95]
+    ... )
+    """
+
+    shape: ShapelyPolygon
+    labels: list[str]
+    scores: list[float] = field(default_factory=list)
+
+    def __post_init__(self):
+        if not isinstance(self.shape, ShapelyPolygon):
+            raise TypeError("shape must be of type shapely.geometry.Polygon.")
+        if self.shape.is_empty:
+            raise ValueError("Polygon is empty.")
+
+        if len(self.scores) == 0 and len(self.labels) != 1:
+            raise ValueError(
+                "Ground truths must be defined with no scores and a single label. If you meant to define a prediction, then please include one score for every label provided."
+            )
+        if len(self.scores) > 0 and len(self.labels) != len(self.scores):
+            raise ValueError(
+                "If scores are defined, there must be a 1:1 pairing with labels."
+            )
+
+    @property
+    def extrema(self) -> tuple[float, float, float, float]:
+        """
+        Returns the polygon's bounding box extrema.
+
+        Returns
+        -------
+        tuple[float, float, float, float]
+            A tuple in the form (xmin, xmax, ymin, ymax).
+        """
+        xmin, ymin, xmax, ymax = self.shape.bounds
+        return (xmin, xmax, ymin, ymax)
+
+    @property
+    def annotation(self) -> ShapelyPolygon:
+        """
+        Returns the annotation's data representation.
+
+        Returns
+        -------
+        shapely.geometry.Polygon
+            The polygon shape.
+        """
+        return self.shape
+
+
+@dataclass
+class Bitmask:
+    """
+    Represents a binary mask with associated labels and optional scores.
+
+    Parameters
+    ----------
+    mask : NDArray[np.bool_]
+        A NumPy array of boolean values representing the mask.
+    labels : list of str
+        List of labels associated with the mask.
+    scores : list of float, optional
+        Confidence scores corresponding to each label. Defaults to an empty list.
+
+    Examples
+    --------
+    Ground Truth Example:
+
+    >>> import numpy as np
+    >>> mask = np.array([[True, False], [False, True]], dtype=np.bool_)
+    >>> bitmask = Bitmask(mask=mask, labels=['tree'])
+
+    Prediction Example:
+
+    >>> bitmask = Bitmask(
+    ...     mask=mask, labels=['tree'], scores=[0.85]
+    ... )
+    """
+
+    mask: NDArray[np.bool_]
+    labels: list[str]
+    scores: list[float] = field(default_factory=list)
+
+    def __post_init__(self):
+
+        if (
+            not isinstance(self.mask, np.ndarray)
+            or self.mask.dtype != np.bool_
+        ):
+            raise ValueError(
+                "Expected mask to be of type `NDArray[np.bool_]`."
+            )
+        elif not self.mask.any():
+            raise ValueError("Mask does not define any object instances.")
+
+        if len(self.scores) == 0 and len(self.labels) != 1:
+            raise ValueError(
+                "Ground truths must be defined with no scores and a single label. If you meant to define a prediction, then please include one score for every label provided."
+            )
+        if len(self.scores) > 0 and len(self.labels) != len(self.scores):
+            raise ValueError(
+                "If scores are defined, there must be a 1:1 pairing with labels."
+            )
+
+    @property
+    def extrema(self) -> tuple[float, float, float, float]:
+        """
+        Returns the bounding box extrema of the mask.
+
+        Returns
+        -------
+        tuple[float, float, float, float]
+            A tuple in the form (xmin, xmax, ymin, ymax).
+        """
+        rows, cols = np.nonzero(self.mask)
+        return (cols.min(), cols.max(), rows.min(), rows.max())
+
+    @property
+    def annotation(self) -> NDArray[np.bool_]:
+        """
+        Returns the annotation's data representation.
+
+        Returns
+        -------
+        NDArray[np.bool_]
+            The binary mask array.
+        """
+        return self.mask
+
+
+@dataclass
+class Detection:
+    """
+    Detection data structure holding ground truths and predictions for object detection tasks.
+
+    Parameters
+    ----------
+    uid : str
+        Unique identifier for the image or sample.
+    groundtruths : list of BoundingBox or Bitmask or Polygon
+        List of ground truth annotations.
+    predictions : list of BoundingBox or Bitmask or Polygon
+        List of predicted annotations.
+
+    Examples
+    --------
+    >>> bbox_gt = BoundingBox(xmin=10, xmax=50, ymin=20, ymax=60, labels=['cat'])
+    >>> bbox_pred = BoundingBox(
+    ...     xmin=12, xmax=48, ymin=22, ymax=58, labels=['cat'], scores=[0.9]
+    ... )
+    >>> detection = Detection(
+    ...     uid='image_001',
+    ...     groundtruths=[bbox_gt],
+    ...     predictions=[bbox_pred]
+    ... )
+    """
+
+    uid: str
+    groundtruths: list[BoundingBox] | list[Bitmask] | list[Polygon]
+    predictions: list[BoundingBox] | list[Bitmask] | list[Polygon]
+
+    def __post_init__(self):
+        for prediction in self.predictions:
+            if len(prediction.scores) != len(prediction.labels):
+                raise ValueError(
+                    "Predictions must provide a score for every label."
+                )