valor-lite 0.33.8__py3-none-any.whl → 0.33.9__py3-none-any.whl

valor_lite/semantic_segmentation/annotation.py

@@ -0,0 +1,96 @@
+from dataclasses import dataclass, field
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+@dataclass
+class Bitmask:
+    """
+    Represents a binary mask with an associated semantic label.
+
+    Parameters
+    ----------
+    mask : NDArray[np.bool_]
+        A NumPy array of boolean values representing the mask.
+    label : str
+        The semantic label associated with the mask.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> mask = np.array([[True, False], [False, True]], dtype=np.bool_)
+    >>> bitmask = Bitmask(mask=mask, label='ocean')
+    """
+
+    mask: NDArray[np.bool_]
+    label: str
+
+    def __post_init__(self):
+        if self.mask.dtype != np.bool_:
+            raise ValueError(
+                f"Bitmask recieved mask with dtype `{self.mask.dtype}`."
+            )
+
+
+@dataclass
+class Segmentation:
+    """
+    Segmentation data structure holding ground truth and prediction bitmasks for semantic segmentation tasks.
+
+    Parameters
+    ----------
+    uid : str
+        Unique identifier for the image or sample.
+    groundtruths : List[Bitmask]
+        List of ground truth bitmasks.
+    predictions : List[Bitmask]
+        List of predicted bitmasks.
+    shape : tuple of int, optional
+        The shape of the segmentation masks. This is set automatically after initialization.
+    size : int, optional
+        The total number of pixels in the masks. This is set automatically after initialization.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> mask1 = np.array([[True, False], [False, True]], dtype=np.bool_)
+    >>> groundtruth = Bitmask(mask=mask1, label='object')
+    >>> mask2 = np.array([[False, True], [True, False]], dtype=np.bool_)
+    >>> prediction = Bitmask(mask=mask2, label='object')
+    >>> segmentation = Segmentation(
+    ...     uid='123',
+    ...     groundtruths=[groundtruth],
+    ...     predictions=[prediction]
+    ... )
+    """
+
+    uid: str
+    groundtruths: list[Bitmask]
+    predictions: list[Bitmask]
+    shape: tuple[int, ...] = field(default_factory=lambda: (0, 0))
+    size: int = field(default=0)
+
+    def __post_init__(self):
+
+        groundtruth_shape = {
+            groundtruth.mask.shape for groundtruth in self.groundtruths
+        }
+        prediction_shape = {
+            prediction.mask.shape for prediction in self.predictions
+        }
+        if len(groundtruth_shape) == 0:
+            raise ValueError("The segmenation is missing ground truths.")
+        elif len(prediction_shape) == 0:
+            raise ValueError("The segmenation is missing predictions.")
+        elif (
+            len(groundtruth_shape) != 1
+            or len(prediction_shape) != 1
+            or groundtruth_shape != prediction_shape
+        ):
+            raise ValueError(
+                "A shape mismatch exists within the segmentation."
+            )
+
+        self.shape = groundtruth_shape.pop()
+        self.size = int(np.prod(np.array(self.shape)))

valor_lite/{segmentation → semantic_segmentation}/manager.py

@@ -4,12 +4,12 @@ from dataclasses import dataclass
 import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
-from valor_lite.segmentation.annotation import Segmentation
-from valor_lite.segmentation.computation import (
+from valor_lite.semantic_segmentation.annotation import Segmentation
+from valor_lite.semantic_segmentation.computation import (
     compute_intermediate_confusion_matrices,
     compute_metrics,
 )
-from valor_lite.segmentation.metric import (
+from valor_lite.semantic_segmentation.metric import (
     F1,
     Accuracy,
     ConfusionMatrix,
@@ -190,9 +190,8 @@ class Evaluator:
             n_pixels=self._n_pixels_per_datum[mask_datums].sum(),
         )
 
-    def evaluate(
+    def compute_precision_recall_iou(
         self,
-        metrics_to_return: list[MetricType] = MetricType.base(),
         filter_: Filter | None = None,
         as_dict: bool = False,
     ) -> dict[MetricType, list]:
@@ -201,10 +200,10 @@ class Evaluator:
 
         Parameters
         ----------
-        metrics_to_return : list[MetricType]
-            A list of metrics to return in the results.
         filter_ : Filter, optional
             An optional filter object.
+        as_dict : bool, default=False
+            An option to return metrics as dictionaries.
 
         Returns
         -------
@@ -239,7 +238,7 @@ class Evaluator:
 
         metrics[MetricType.Accuracy] = [
             Accuracy(
-                value=accuracy,
+                value=float(accuracy),
             )
         ]
 
@@ -258,16 +257,14 @@ class Evaluator:
                 },
                 hallucinations={
                     self.index_to_label[pd_label_idx]: {
-                        "percent": float(hallucination_ratios[pd_label_idx])
+                        "ratio": float(hallucination_ratios[pd_label_idx])
                     }
                     for pd_label_idx in range(self.n_labels)
                     if label_metadata[pd_label_idx, 0] > 0
                 },
                 missing_predictions={
                     self.index_to_label[gt_label_idx]: {
-                        "percent": float(
-                            missing_prediction_ratios[gt_label_idx]
-                        )
+                        "ratio": float(missing_prediction_ratios[gt_label_idx])
                     }
                     for gt_label_idx in range(self.n_labels)
                     if label_metadata[gt_label_idx, 0] > 0
@@ -316,10 +313,6 @@ class Evaluator:
                 )
             )
 
-        for metric in set(metrics.keys()):
-            if metric not in metrics_to_return:
-                del metrics[metric]
-
         if as_dict:
             return {
                 mtype: [metric.to_dict() for metric in mvalues]
@@ -328,6 +321,30 @@ class Evaluator:
 
         return metrics
 
+    def evaluate(
+        self,
+        filter_: Filter | None = None,
+        as_dict: bool = False,
+    ) -> dict[MetricType, list]:
+        """
+        Computes all available metrics.
+
+        Parameters
+        ----------
+        filter_ : Filter, optional
+            An optional filter object.
+        as_dict : bool, default=False
+            An option to return metrics as dictionaries.
+
+        Returns
+        -------
+        dict[MetricType, list]
+            A dictionary mapping metric type to lists of metrics.
+        """
+        return self.compute_precision_recall_iou(
+            filter_=filter_, as_dict=as_dict
+        )
+
 
 class DataLoader:
     """

valor_lite/semantic_segmentation/metric.py

@@ -0,0 +1,278 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from valor_lite.schemas import Metric
+
+
+class MetricType(Enum):
+    Precision = "Precision"
+    Recall = "Recall"
+    Accuracy = "Accuracy"
+    F1 = "F1"
+    IoU = "IoU"
+    mIoU = "mIoU"
+    ConfusionMatrix = "ConfusionMatrix"
+
+    @classmethod
+    def base(cls):
+        return [
+            cls.Precision,
+            cls.Recall,
+            cls.Accuracy,
+            cls.F1,
+            cls.IoU,
+            cls.mIoU,
+            cls.ConfusionMatrix,
+        ]
+
+
+@dataclass
+class _LabelValue:
+    value: float
+    label: str
+
+    def to_metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value=self.value,
+            parameters={
+                "label": self.label,
+            },
+        )
+
+    def to_dict(self) -> dict:
+        return self.to_metric().to_dict()
+
+
+class Precision(_LabelValue):
+    """
+    Precision metric for a specific class label.
+
+    Precision is calulated using the number of true-positive pixels divided by
+    the sum of all true-positive and false-positive pixels.
+
+    Attributes
+    ----------
+    value : float
+        The computed precision value.
+    label : str
+        The label for which the precision is calculated.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+class Recall(_LabelValue):
+    """
+    Recall metric for a specific class label.
+
+    Recall is calulated using the number of true-positive pixels divided by
+    the sum of all true-positive and false-negative pixels.
+
+    Attributes
+    ----------
+    value : float
+        The computed recall value.
+    label : str
+        The label for which the recall is calculated.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+class F1(_LabelValue):
+    """
+    F1 score for a specific class label.
+
+    Attributes
+    ----------
+    value : float
+        The computed F1 score.
+    label : str
+        The label for which the F1 score is calculated.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+class IoU(_LabelValue):
+    """
+    Intersection over Union (IoU) ratio for a specific class label.
+
+    Attributes
+    ----------
+    value : float
+        The computed IoU ratio.
+    label : str
+        The label for which the IoU is calculated.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+@dataclass
+class _Value:
+    value: float
+
+    def to_metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value=self.value,
+            parameters={},
+        )
+
+    def to_dict(self) -> dict:
+        return self.to_metric().to_dict()
+
+
+class Accuracy(_Value):
+    """
+    Accuracy metric computed over all labels.
+
+    Attributes
+    ----------
+    value : float
+        The accuracy value.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+class mIoU(_Value):
+    """
+    Mean Intersection over Union (mIoU) ratio.
+
+    The mIoU value is computed by averaging IoU over all labels.
+
+    Attributes
+    ----------
+    value : float
+        The mIoU value.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+@dataclass
+class ConfusionMatrix:
+    """
+    The confusion matrix and related metrics for semantic segmentation tasks.
+
+    This class encapsulates detailed information about the model's performance, including correct
+    predictions, misclassifications, hallucinations (false positives), and missing predictions
+    (false negatives). It provides counts for each category to facilitate in-depth analysis.
+
+    Confusion Matrix Format:
+    {
+        <ground truth label>: {
+            <prediction label>: {
+                'iou': <float>,
+            },
+        },
+    }
+
+    Hallucinations Format:
+    {
+        <prediction label>: {
+            'iou': <float>,
+        },
+    }
+
+    Missing Predictions Format:
+    {
+        <ground truth label>: {
+            'iou': <float>,
+        },
+    }
+
+    Attributes
+    ----------
+    confusion_matrix : dict
+        Nested dictionaries representing the Intersection over Union (IoU) scores for each
+        ground truth label and prediction label pair.
+    hallucinations : dict
+        Dictionary representing the pixel ratios for predicted labels that do not correspond
+        to any ground truth labels (false positives).
+    missing_predictions : dict
+        Dictionary representing the pixel ratios for ground truth labels that were not predicted
+        (false negatives).
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    confusion_matrix: dict[
+        str,  # ground truth label value
+        dict[
+            str,  # prediction label value
+            dict[str, float],  # iou
+        ],
+    ]
+    hallucinations: dict[
+        str,  # prediction label value
+        dict[str, float],  # pixel ratio
+    ]
+    missing_predictions: dict[
+        str,  # ground truth label value
+        dict[str, float],  # pixel ratio
+    ]
+
+    def to_metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value={
+                "confusion_matrix": self.confusion_matrix,
+                "hallucinations": self.hallucinations,
+                "missing_predictions": self.missing_predictions,
+            },
+            parameters={},
+        )
+
+    def to_dict(self) -> dict:
+        return self.to_metric().to_dict()

valor_lite-0.33.9.dist-info/METADATA

@@ -0,0 +1,179 @@
+Metadata-Version: 2.1
+Name: valor-lite
+Version: 0.33.9
+Summary: Compute valor metrics locally.
+License: MIT License
+        
+        Copyright (c) 2023 Striveworks
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: homepage, https://www.striveworks.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: Pillow >=9.1.0
+Requires-Dist: tqdm
+Requires-Dist: requests
+Requires-Dist: numpy
+Requires-Dist: shapely
+Requires-Dist: importlib-metadata ; python_version < "3.8"
+Provides-Extra: test
+Requires-Dist: pytest ; extra == 'test'
+Requires-Dist: coverage ; extra == 'test'
+
+# valor-lite: Fast, local machine learning evaluation.
+
+valor-lite is a lightweight, numpy-based library designed for fast and seamless evaluation of machine learning models. It is optimized for environments where quick, responsive evaluations are essential, whether as part of a larger service or embedded within user-facing tools.
+
+valor-lite is maintained by Striveworks, a cutting-edge MLOps company based in Austin, Texas. If you'd like to learn more or have questions, we invite you to connect with us on [Slack](https://striveworks-public.slack.com/join/shared_invite/zt-1a0jx768y-2J1fffN~b4fXYM8GecvOhA#/shared-invite/email) or explore our [GitHub repository](https://github.com/striveworks/valor).
+
+For additional details, be sure to check out our user [documentation](https://striveworks.github.io/valor/). We're excited to support you in making the most of Valor!
+
+## Usage
+
+### Classification
+
+```python
+from valor_lite.classification import DataLoader, Classification, MetricType
+
+classifications = [
+    Classification(
+        uid="uid0",
+        groundtruth="dog",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.75, 0.2, 0.05],
+    ),
+    Classification(
+        uid="uid1",
+        groundtruth="cat",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.41, 0.39, 0.1],
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(classifications)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': [0.5],
+    'parameters': {
+        'score_thresholds': [0.0],
+        'hardmax': True,
+        'label': 'dog'
+    }
+}
+```
+
+### Object Detection
+
+```python
+from valor_lite.object_detection import DataLoader, Detection, BoundingBox, MetricType
+
+detections = [
+    Detection(
+        uid="uid0",
+        groundtruths=[
+            BoundingBox(
+                xmin=0, xmax=10,
+                ymin=0, ymax=10,
+                labels=["dog"]
+            ),
+            BoundingBox(
+                xmin=20, xmax=30,
+                ymin=20, ymax=30,
+                labels=["cat"]
+            ),
+        ],
+        predictions=[
+            BoundingBox(
+                xmin=1, xmax=11,
+                ymin=1, ymax=11,
+                labels=["dog", "cat", "bird"],
+                scores=[0.85, 0.1, 0.05]
+            ),
+            BoundingBox(
+                xmin=21, xmax=31,
+                ymin=21, ymax=31,
+                labels=["dog", "cat", "bird"],
+                scores=[0.34, 0.33, 0.33]
+            ),
+        ],
+    ),
+]
+
+loader = DataLoader()
+loader.add_bounding_boxes(detections)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': 0.5,
+    'parameters': {
+        'iou_threshold': 0.5,
+        'score_threshold': 0.5,
+        'label': 'dog'
+    }
+}
+```
+
+### Semantic Segmentation
+
+```python
+import numpy as np
+from valor_lite.semantic_segmentation import DataLoader, Segmentation, Bitmask, MetricType
+
+segmentations = [
+    Segmentation(
+        uid="uid0",
+        groundtruths=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ],
+        predictions=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ]
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(segmentations)
+evaluator = loader.finalize()
+
+print(metrics[MetricType.Precision][0])
+```

valor_lite-0.33.9.dist-info/RECORD

@@ -0,0 +1,24 @@
+valor_lite/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
+valor_lite/classification/__init__.py,sha256=2wmmziIzUATm7MbmAcPNLXrEX5l4oeD7XBwPd9bWM3Q,506
+valor_lite/classification/annotation.py,sha256=0aUOvcwBAZgiNOJuyh-pXyNTG7vP7r8CUfnU3OmpUwQ,1113
+valor_lite/classification/computation.py,sha256=qd9K7CcSGmMm_7shfX47_ZIuB-uE2LLiLMZSS_3NJTk,12093
+valor_lite/classification/manager.py,sha256=7NKk4syQHH5hBEUDWTD0zIFkJSNdOMzJn8a8GzfBnDc,23205
+valor_lite/classification/metric.py,sha256=m9_zD82YGl0QhuMql943YNKg67NZ6bsrR8ggs6_JZms,11728
+valor_lite/object_detection/__init__.py,sha256=PiKfemo8FkZRzBhPSjhil8ahGURLy0Vk_iV25CB4UBU,1139
+valor_lite/object_detection/annotation.py,sha256=o6VfiRobiB0ljqsNBLAYMXgi32RSIR7uTA-dgxq6zBI,8248
+valor_lite/object_detection/computation.py,sha256=7rOfVlYDadXcJ1_S0FJRF3IPigcsR7guk_0rXeIdAOE,26919
+valor_lite/object_detection/manager.py,sha256=k8VRqmlfWGKj1IuijbG49jXkMelE8v59pTQTCwkSMKk,38833
+valor_lite/object_detection/metric.py,sha256=nWSqIQSBQrpl3Stz_xe2-AYoo2nrATeMuFVFmREjSNA,23833
+valor_lite/semantic_segmentation/__init__.py,sha256=IdarTHKUuUMDvMBmInQu12Mm_NMCbql6Hf0nL5b56Ak,424
+valor_lite/semantic_segmentation/annotation.py,sha256=CujYFdHS3fgr4Y7mEDs_u1XBmbPJzNU2CdqvjCT_d_A,2938
+valor_lite/semantic_segmentation/computation.py,sha256=iJkEmTNmw9HwQCxSnpJkQsAdVcFriGhhu_WMks6D7tU,5122
+valor_lite/semantic_segmentation/manager.py,sha256=aJk6edWZWKqrzl6hVmEUSZVYhHLuyihxWgAIXsCXkZ0,17361
+valor_lite/semantic_segmentation/metric.py,sha256=Y8M3z92SaABEe9TwBUN37TFsh9DR5WoIxO-TfXVwz8I,6289
+valor_lite/text_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite-0.33.9.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite-0.33.9.dist-info/METADATA,sha256=dXS7Nt_WHKBaIARWZ3Ek27i26-pWyatewb3eFEnYor8,5631
+valor_lite-0.33.9.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+valor_lite-0.33.9.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.33.9.dist-info/RECORD,,