valor-lite 0.33.15__py3-none-any.whl → 0.33.17__py3-none-any.whl

valor_lite/classification/metric.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from enum import Enum
 
 from valor_lite.schemas import BaseMetric
@@ -14,6 +15,7 @@ class MetricType(Enum):
     ConfusionMatrix = "ConfusionMatrix"
 
 
+@dataclass
 class Metric(BaseMetric):
     """
     Classification Metric.
@@ -28,6 +30,24 @@ class Metric(BaseMetric):
         A dictionary containing metric parameters.
     """
 
+    def __post_init__(self):
+        if not isinstance(self.type, str):
+            raise TypeError(
+                f"Metric type should be of type 'str': {self.type}"
+            )
+        elif not isinstance(self.value, (int, float, dict)):
+            raise TypeError(
+                f"Metric value must be of type 'int', 'float' or 'dict': {self.value}"
+            )
+        elif not isinstance(self.parameters, dict):
+            raise TypeError(
+                f"Metric parameters must be of type 'dict[str, Any]': {self.parameters}"
+            )
+        elif not all([isinstance(k, str) for k in self.parameters.keys()]):
+            raise TypeError(
+                f"Metric parameter dictionary should only have keys with type 'str': {self.parameters}"
+            )
+
     @classmethod
     def precision(
         cls,

valor_lite/object_detection/annotation.py

@@ -142,18 +142,6 @@ class Polygon:
         xmin, ymin, xmax, ymax = self.shape.bounds
         return (xmin, xmax, ymin, ymax)
 
-    @property
-    def annotation(self) -> ShapelyPolygon:
-        """
-        Returns the annotation's data representation.
-
-        Returns
-        -------
-        shapely.geometry.Polygon
-            The polygon shape.
-        """
-        return self.shape
-
 
 @dataclass
 class Bitmask:
@@ -222,18 +210,6 @@ class Bitmask:
         rows, cols = np.nonzero(self.mask)
         return (cols.min(), cols.max(), rows.min(), rows.max())
 
-    @property
-    def annotation(self) -> NDArray[np.bool_]:
-        """
-        Returns the annotation's data representation.
-
-        Returns
-        -------
-        NDArray[np.bool_]
-            The binary mask array.
-        """
-        return self.mask
-
 
 @dataclass
 class Detection:

valor_lite/object_detection/manager.py

@@ -1,17 +1,10 @@
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Type
 
 import numpy as np
-import valor_lite.object_detection.annotation as annotation
 from numpy.typing import NDArray
 from tqdm import tqdm
-from valor_lite.object_detection.annotation import (
-    Bitmask,
-    BoundingBox,
-    Detection,
-    Polygon,
-)
+from valor_lite.object_detection.annotation import Detection
 from valor_lite.object_detection.computation import (
     compute_bbox_iou,
     compute_bitmask_iou,
@@ -307,7 +300,7 @@ class Evaluator:
         filter_: Filter | None = None,
     ) -> dict[MetricType, list[Metric]]:
         """
-        Computes all avaiable metrics.
+        Computes all available metrics.
 
         Parameters
         ----------
@@ -396,74 +389,47 @@ class DataLoader:
 
         return self._evaluator.label_to_index[label]
 
-    def _compute_ious_and_cache_pairs(
+    def _cache_pairs(
         self,
         uid_index: int,
         groundtruths: list,
         predictions: list,
-        annotation_type: Type[BoundingBox] | Type[Polygon] | Type[Bitmask],
+        ious: NDArray[np.float64],
     ) -> None:
         """
         Compute IOUs between groundtruths and preditions before storing as pairs.
 
         Parameters
         ----------
-        uid_index: int
+        uid_index : int
             The index of the detection.
-        groundtruths: list
+        groundtruths : list
             A list of groundtruths.
-        predictions: list
+        predictions : list
             A list of predictions.
-        annotation_type: type[BoundingBox] | type[Polygon] | type[Bitmask]
-            The type of annotation to compute IOUs for.
+        ious : NDArray[np.float64]
+            An array with shape (n_preds, n_gts) containing IOUs.
         """
 
-        pairs = list()
-        n_predictions = len(predictions)
-        n_groundtruths = len(groundtruths)
-
-        all_pairs = np.array(
-            [
-                np.array([gann, pann])
-                for _, _, _, pann in predictions
-                for _, _, gann in groundtruths
-            ]
-        )
-
-        match annotation_type:
-            case annotation.BoundingBox:
-                ious = compute_bbox_iou(all_pairs)
-            case annotation.Polygon:
-                ious = compute_polygon_iou(all_pairs)
-            case annotation.Bitmask:
-                ious = compute_bitmask_iou(all_pairs)
-            case _:
-                raise ValueError(
-                    f"Invalid annotation type `{annotation_type}`."
-                )
-
-        ious = ious.reshape(n_predictions, n_groundtruths)
         predictions_with_iou_of_zero = np.where((ious < 1e-9).all(axis=1))[0]
         groundtruths_with_iou_of_zero = np.where((ious < 1e-9).all(axis=0))[0]
 
-        pairs.extend(
-            [
-                np.array(
-                    [
-                        float(uid_index),
-                        float(gidx),
-                        float(pidx),
-                        ious[pidx, gidx],
-                        float(glabel),
-                        float(plabel),
-                        float(score),
-                    ]
-                )
-                for pidx, plabel, score, _ in predictions
-                for gidx, glabel, _ in groundtruths
-                if ious[pidx, gidx] >= 1e-9
-            ]
-        )
+        pairs = [
+            np.array(
+                [
+                    float(uid_index),
+                    float(gidx),
+                    float(pidx),
+                    ious[pidx, gidx],
+                    float(glabel),
+                    float(plabel),
+                    float(score),
+                ]
+            )
+            for pidx, plabel, score in predictions
+            for gidx, glabel in groundtruths
+            if ious[pidx, gidx] >= 1e-9
+        ]
         pairs.extend(
             [
                 np.array(
@@ -496,13 +462,12 @@ class DataLoader:
                 for index in groundtruths_with_iou_of_zero
             ]
         )
-
         self.pairs.append(np.array(pairs))
 
     def _add_data(
         self,
         detections: list[Detection],
-        annotation_type: type[Bitmask] | type[BoundingBox] | type[Polygon],
+        detection_ious: list[NDArray[np.float64]],
         show_progress: bool = False,
     ):
         """
@@ -512,13 +477,15 @@ class DataLoader:
         ----------
         detections : list[Detection]
             A list of Detection objects.
-        annotation_type : type[Bitmask] | type[BoundingBox] | type[Polygon]
-            The annotation type to process.
+        detection_ious : list[NDArray[np.float64]]
+            A list of arrays containing IOUs per detection.
         show_progress : bool, default=False
             Toggle for tqdm progress bar.
         """
         disable_tqdm = not show_progress
-        for detection in tqdm(detections, disable=disable_tqdm):
+        for detection, ious in tqdm(
+            zip(detections, detection_ious), disable=disable_tqdm
+        ):
 
             # update metadata
             self._evaluator.n_datums += 1
@@ -541,11 +508,6 @@ class DataLoader:
             predictions = list()
 
             for gidx, gann in enumerate(detection.groundtruths):
-                if not isinstance(gann, annotation_type):
-                    raise ValueError(
-                        f"Expected {annotation_type}, but annotation is of type {type(gann)}."
-                    )
-
                 self._evaluator.groundtruth_examples[uid_index][
                     gidx
                 ] = gann.extrema
@@ -556,16 +518,10 @@ class DataLoader:
                         (
                             gidx,
                             label_idx,
-                            gann.annotation,
                         )
                     )
 
             for pidx, pann in enumerate(detection.predictions):
-                if not isinstance(pann, annotation_type):
-                    raise ValueError(
-                        f"Expected {annotation_type}, but annotation is of type {type(pann)}."
-                    )
-
                 self._evaluator.prediction_examples[uid_index][
                     pidx
                 ] = pann.extrema
@@ -577,15 +533,14 @@ class DataLoader:
                             pidx,
                             label_idx,
                             pscore,
-                            pann.annotation,
                         )
                     )
 
-            self._compute_ious_and_cache_pairs(
+            self._cache_pairs(
                 uid_index=uid_index,
                 groundtruths=groundtruths,
                 predictions=predictions,
-                annotation_type=annotation_type,
+                ious=ious,
             )
 
     def add_bounding_boxes(
@@ -603,10 +558,22 @@ class DataLoader:
         show_progress : bool, default=False
             Toggle for tqdm progress bar.
         """
+        ious = [
+            compute_bbox_iou(
+                np.array(
+                    [
+                        [gt.extrema, pd.extrema]
+                        for pd in detection.predictions
+                        for gt in detection.groundtruths
+                    ]
+                )
+            ).reshape(len(detection.predictions), len(detection.groundtruths))
+            for detection in detections
+        ]
         return self._add_data(
             detections=detections,
+            detection_ious=ious,
             show_progress=show_progress,
-            annotation_type=BoundingBox,
         )
 
     def add_polygons(
@@ -624,10 +591,22 @@ class DataLoader:
         show_progress : bool, default=False
             Toggle for tqdm progress bar.
         """
+        ious = [
+            compute_polygon_iou(
+                np.array(
+                    [
+                        [gt.shape, pd.shape]  # type: ignore - using the AttributeError as a validator
+                        for pd in detection.predictions
+                        for gt in detection.groundtruths
+                    ]
+                )
+            ).reshape(len(detection.predictions), len(detection.groundtruths))
+            for detection in detections
+        ]
         return self._add_data(
             detections=detections,
+            detection_ious=ious,
             show_progress=show_progress,
-            annotation_type=Polygon,
         )
 
     def add_bitmasks(
@@ -645,10 +624,22 @@ class DataLoader:
         show_progress : bool, default=False
             Toggle for tqdm progress bar.
         """
+        ious = [
+            compute_bitmask_iou(
+                np.array(
+                    [
+                        [gt.mask, pd.mask]  # type: ignore - using the AttributeError as a validator
+                        for pd in detection.predictions
+                        for gt in detection.groundtruths
+                    ]
+                )
+            ).reshape(len(detection.predictions), len(detection.groundtruths))
+            for detection in detections
+        ]
         return self._add_data(
             detections=detections,
+            detection_ious=ious,
             show_progress=show_progress,
-            annotation_type=Bitmask,
         )
 
     def finalize(self) -> Evaluator:

valor_lite/object_detection/metric.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from enum import Enum
 
 from valor_lite.schemas import BaseMetric
@@ -21,6 +22,7 @@ class MetricType(str, Enum):
     ConfusionMatrix = "ConfusionMatrix"
 
 
+@dataclass
 class Metric(BaseMetric):
     """
     Object Detection Metric.
@@ -35,6 +37,24 @@ class Metric(BaseMetric):
         A dictionary containing metric parameters.
     """
 
+    def __post_init__(self):
+        if not isinstance(self.type, str):
+            raise TypeError(
+                f"Metric type should be of type 'str': {self.type}"
+            )
+        elif not isinstance(self.value, (int, float, dict)):
+            raise TypeError(
+                f"Metric value must be of type 'int', 'float' or 'dict': {self.value}"
+            )
+        elif not isinstance(self.parameters, dict):
+            raise TypeError(
+                f"Metric parameters must be of type 'dict[str, Any]': {self.parameters}"
+            )
+        elif not all([isinstance(k, str) for k in self.parameters.keys()]):
+            raise TypeError(
+                f"Metric parameter dictionary should only have keys with type 'str': {self.parameters}"
+            )
+
     @classmethod
     def precision(
         cls,

valor_lite/schemas.py

@@ -7,11 +7,5 @@ class BaseMetric:
     value: int | float | dict
     parameters: dict
 
-    def __post_init__(self):
-        if not isinstance(self.value, (int, float, dict)):
-            raise TypeError(
-                "Metric value must be of type `int`, `float` or `dict`."
-            )
-
     def to_dict(self) -> dict:
         return asdict(self)

valor_lite/semantic_segmentation/computation.py

@@ -46,8 +46,8 @@ def compute_intermediate_confusion_matrices(
         predictions.reshape(1, n_pd_labels, -1),
     ).sum(axis=2)
 
-    intersected_groundtruth_counts = intersection_counts.sum(axis=0)
-    intersected_prediction_counts = intersection_counts.sum(axis=1)
+    intersected_groundtruth_counts = intersection_counts.sum(axis=1)
+    intersected_prediction_counts = intersection_counts.sum(axis=0)
 
     confusion_matrix = np.zeros((n_labels + 1, n_labels + 1), dtype=np.int32)
     confusion_matrix[0, 0] = background_counts

valor_lite/semantic_segmentation/metric.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from enum import Enum
 
 from valor_lite.schemas import BaseMetric
@@ -13,6 +14,7 @@ class MetricType(Enum):
     ConfusionMatrix = "ConfusionMatrix"
 
 
+@dataclass
 class Metric(BaseMetric):
     """
     Semantic Segmentation Metric.
@@ -27,6 +29,24 @@ class Metric(BaseMetric):
         A dictionary containing metric parameters.
     """
 
+    def __post_init__(self):
+        if not isinstance(self.type, str):
+            raise TypeError(
+                f"Metric type should be of type 'str': {self.type}"
+            )
+        elif not isinstance(self.value, (int, float, dict)):
+            raise TypeError(
+                f"Metric value must be of type 'int', 'float' or 'dict': {self.value}"
+            )
+        elif not isinstance(self.parameters, dict):
+            raise TypeError(
+                f"Metric parameters must be of type 'dict[str, Any]': {self.parameters}"
+            )
+        elif not all([isinstance(k, str) for k in self.parameters.keys()]):
+            raise TypeError(
+                f"Metric parameter dictionary should only have keys with type 'str': {self.parameters}"
+            )
+
     @classmethod
     def precision(
         cls,

valor_lite/text_generation/__init__.py

@@ -0,0 +1,15 @@
+from .annotation import Context, QueryResponse
+from .llm.integrations import ClientWrapper, MistralWrapper, OpenAIWrapper
+from .manager import Evaluator
+from .metric import Metric, MetricType
+
+__all__ = [
+    "QueryResponse",
+    "Context",
+    "Evaluator",
+    "Metric",
+    "MetricType",
+    "ClientWrapper",
+    "OpenAIWrapper",
+    "MistralWrapper",
+]

valor_lite/text_generation/annotation.py

@@ -0,0 +1,56 @@
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Context:
+    """
+    Contextual ground truth and prediction.
+
+    Attributes
+    ----------
+    groundtruth : list[str]
+        The definitive context.
+    prediction : list[str]
+        Any retrieved context from a retrieval-augmented-generation (RAG) pipeline.
+
+    Examples
+    --------
+    ... context = Context(
+    ...     groundtruth=[...],
+    ...     prediction=[...],
+    ... )
+    """
+
+    groundtruth: list[str] = field(default_factory=list)
+    prediction: list[str] = field(default_factory=list)
+
+
+@dataclass
+class QueryResponse:
+    """
+    Text generation data structure containing ground truths and predictions.
+
+    Attributes
+    ----------
+    query : str
+        The user query.
+    response : str
+        The language model's response.
+    context : Context
+        Any context provided to the model.
+
+    Examples
+    --------
+    >>> query = QueryResponse(
+    ...     query='When was George Washington born?',
+    ...     response="February 22, 1732",
+    ...     context=Context(
+    ...         groundtruth=["02/22/1732"],
+    ...         prediction=["02/22/1732"],
+    ...     ),
+    ... )
+    """
+
+    query: str
+    response: str
+    context: Context | None = field(default=None)