valor-lite 0.33.7__py3-none-any.whl → 0.33.9__py3-none-any.whl

valor_lite/semantic_segmentation/metric.py

@@ -0,0 +1,278 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from valor_lite.schemas import Metric
+
+
+class MetricType(Enum):
+    Precision = "Precision"
+    Recall = "Recall"
+    Accuracy = "Accuracy"
+    F1 = "F1"
+    IoU = "IoU"
+    mIoU = "mIoU"
+    ConfusionMatrix = "ConfusionMatrix"
+
+    @classmethod
+    def base(cls):
+        return [
+            cls.Precision,
+            cls.Recall,
+            cls.Accuracy,
+            cls.F1,
+            cls.IoU,
+            cls.mIoU,
+            cls.ConfusionMatrix,
+        ]
+
+
+@dataclass
+class _LabelValue:
+    value: float
+    label: str
+
+    def to_metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value=self.value,
+            parameters={
+                "label": self.label,
+            },
+        )
+
+    def to_dict(self) -> dict:
+        return self.to_metric().to_dict()
+
+
+class Precision(_LabelValue):
+    """
+    Precision metric for a specific class label.
+
+    Precision is calulated using the number of true-positive pixels divided by
+    the sum of all true-positive and false-positive pixels.
+
+    Attributes
+    ----------
+    value : float
+        The computed precision value.
+    label : str
+        The label for which the precision is calculated.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+class Recall(_LabelValue):
+    """
+    Recall metric for a specific class label.
+
+    Recall is calulated using the number of true-positive pixels divided by
+    the sum of all true-positive and false-negative pixels.
+
+    Attributes
+    ----------
+    value : float
+        The computed recall value.
+    label : str
+        The label for which the recall is calculated.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+class F1(_LabelValue):
+    """
+    F1 score for a specific class label.
+
+    Attributes
+    ----------
+    value : float
+        The computed F1 score.
+    label : str
+        The label for which the F1 score is calculated.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+class IoU(_LabelValue):
+    """
+    Intersection over Union (IoU) ratio for a specific class label.
+
+    Attributes
+    ----------
+    value : float
+        The computed IoU ratio.
+    label : str
+        The label for which the IoU is calculated.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+@dataclass
+class _Value:
+    value: float
+
+    def to_metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value=self.value,
+            parameters={},
+        )
+
+    def to_dict(self) -> dict:
+        return self.to_metric().to_dict()
+
+
+class Accuracy(_Value):
+    """
+    Accuracy metric computed over all labels.
+
+    Attributes
+    ----------
+    value : float
+        The accuracy value.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+class mIoU(_Value):
+    """
+    Mean Intersection over Union (mIoU) ratio.
+
+    The mIoU value is computed by averaging IoU over all labels.
+
+    Attributes
+    ----------
+    value : float
+        The mIoU value.
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    pass
+
+
+@dataclass
+class ConfusionMatrix:
+    """
+    The confusion matrix and related metrics for semantic segmentation tasks.
+
+    This class encapsulates detailed information about the model's performance, including correct
+    predictions, misclassifications, hallucinations (false positives), and missing predictions
+    (false negatives). It provides counts for each category to facilitate in-depth analysis.
+
+    Confusion Matrix Format:
+    {
+        <ground truth label>: {
+            <prediction label>: {
+                'iou': <float>,
+            },
+        },
+    }
+
+    Hallucinations Format:
+    {
+        <prediction label>: {
+            'iou': <float>,
+        },
+    }
+
+    Missing Predictions Format:
+    {
+        <ground truth label>: {
+            'iou': <float>,
+        },
+    }
+
+    Attributes
+    ----------
+    confusion_matrix : dict
+        Nested dictionaries representing the Intersection over Union (IoU) scores for each
+        ground truth label and prediction label pair.
+    hallucinations : dict
+        Dictionary representing the pixel ratios for predicted labels that do not correspond
+        to any ground truth labels (false positives).
+    missing_predictions : dict
+        Dictionary representing the pixel ratios for ground truth labels that were not predicted
+        (false negatives).
+
+    Methods
+    -------
+    to_metric()
+        Converts the instance to a generic `Metric` object.
+    to_dict()
+        Converts the instance to a dictionary representation.
+    """
+
+    confusion_matrix: dict[
+        str,  # ground truth label value
+        dict[
+            str,  # prediction label value
+            dict[str, float],  # iou
+        ],
+    ]
+    hallucinations: dict[
+        str,  # prediction label value
+        dict[str, float],  # pixel ratio
+    ]
+    missing_predictions: dict[
+        str,  # ground truth label value
+        dict[str, float],  # pixel ratio
+    ]
+
+    def to_metric(self) -> Metric:
+        return Metric(
+            type=type(self).__name__,
+            value={
+                "confusion_matrix": self.confusion_matrix,
+                "hallucinations": self.hallucinations,
+                "missing_predictions": self.missing_predictions,
+            },
+            parameters={},
+        )
+
+    def to_dict(self) -> dict:
+        return self.to_metric().to_dict()

valor_lite-0.33.9.dist-info/METADATA

@@ -0,0 +1,179 @@
+Metadata-Version: 2.1
+Name: valor-lite
+Version: 0.33.9
+Summary: Compute valor metrics locally.
+License: MIT License
+        
+        Copyright (c) 2023 Striveworks
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: homepage, https://www.striveworks.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: Pillow >=9.1.0
+Requires-Dist: tqdm
+Requires-Dist: requests
+Requires-Dist: numpy
+Requires-Dist: shapely
+Requires-Dist: importlib-metadata ; python_version < "3.8"
+Provides-Extra: test
+Requires-Dist: pytest ; extra == 'test'
+Requires-Dist: coverage ; extra == 'test'
+
+# valor-lite: Fast, local machine learning evaluation.
+
+valor-lite is a lightweight, numpy-based library designed for fast and seamless evaluation of machine learning models. It is optimized for environments where quick, responsive evaluations are essential, whether as part of a larger service or embedded within user-facing tools.
+
+valor-lite is maintained by Striveworks, a cutting-edge MLOps company based in Austin, Texas. If you'd like to learn more or have questions, we invite you to connect with us on [Slack](https://striveworks-public.slack.com/join/shared_invite/zt-1a0jx768y-2J1fffN~b4fXYM8GecvOhA#/shared-invite/email) or explore our [GitHub repository](https://github.com/striveworks/valor).
+
+For additional details, be sure to check out our user [documentation](https://striveworks.github.io/valor/). We're excited to support you in making the most of Valor!
+
+## Usage
+
+### Classification
+
+```python
+from valor_lite.classification import DataLoader, Classification, MetricType
+
+classifications = [
+    Classification(
+        uid="uid0",
+        groundtruth="dog",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.75, 0.2, 0.05],
+    ),
+    Classification(
+        uid="uid1",
+        groundtruth="cat",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.41, 0.39, 0.1],
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(classifications)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': [0.5],
+    'parameters': {
+        'score_thresholds': [0.0],
+        'hardmax': True,
+        'label': 'dog'
+    }
+}
+```
+
+### Object Detection
+
+```python
+from valor_lite.object_detection import DataLoader, Detection, BoundingBox, MetricType
+
+detections = [
+    Detection(
+        uid="uid0",
+        groundtruths=[
+            BoundingBox(
+                xmin=0, xmax=10,
+                ymin=0, ymax=10,
+                labels=["dog"]
+            ),
+            BoundingBox(
+                xmin=20, xmax=30,
+                ymin=20, ymax=30,
+                labels=["cat"]
+            ),
+        ],
+        predictions=[
+            BoundingBox(
+                xmin=1, xmax=11,
+                ymin=1, ymax=11,
+                labels=["dog", "cat", "bird"],
+                scores=[0.85, 0.1, 0.05]
+            ),
+            BoundingBox(
+                xmin=21, xmax=31,
+                ymin=21, ymax=31,
+                labels=["dog", "cat", "bird"],
+                scores=[0.34, 0.33, 0.33]
+            ),
+        ],
+    ),
+]
+
+loader = DataLoader()
+loader.add_bounding_boxes(detections)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': 0.5,
+    'parameters': {
+        'iou_threshold': 0.5,
+        'score_threshold': 0.5,
+        'label': 'dog'
+    }
+}
+```
+
+### Semantic Segmentation
+
+```python
+import numpy as np
+from valor_lite.semantic_segmentation import DataLoader, Segmentation, Bitmask, MetricType
+
+segmentations = [
+    Segmentation(
+        uid="uid0",
+        groundtruths=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ],
+        predictions=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ]
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(segmentations)
+evaluator = loader.finalize()
+
+print(metrics[MetricType.Precision][0])
+```

valor_lite-0.33.9.dist-info/RECORD

@@ -0,0 +1,24 @@
+valor_lite/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/schemas.py,sha256=r4cC10w1xYsA785KmGE4ePeOX3wzEs846vT7QAiVg_I,293
+valor_lite/classification/__init__.py,sha256=2wmmziIzUATm7MbmAcPNLXrEX5l4oeD7XBwPd9bWM3Q,506
+valor_lite/classification/annotation.py,sha256=0aUOvcwBAZgiNOJuyh-pXyNTG7vP7r8CUfnU3OmpUwQ,1113
+valor_lite/classification/computation.py,sha256=qd9K7CcSGmMm_7shfX47_ZIuB-uE2LLiLMZSS_3NJTk,12093
+valor_lite/classification/manager.py,sha256=7NKk4syQHH5hBEUDWTD0zIFkJSNdOMzJn8a8GzfBnDc,23205
+valor_lite/classification/metric.py,sha256=m9_zD82YGl0QhuMql943YNKg67NZ6bsrR8ggs6_JZms,11728
+valor_lite/object_detection/__init__.py,sha256=PiKfemo8FkZRzBhPSjhil8ahGURLy0Vk_iV25CB4UBU,1139
+valor_lite/object_detection/annotation.py,sha256=o6VfiRobiB0ljqsNBLAYMXgi32RSIR7uTA-dgxq6zBI,8248
+valor_lite/object_detection/computation.py,sha256=7rOfVlYDadXcJ1_S0FJRF3IPigcsR7guk_0rXeIdAOE,26919
+valor_lite/object_detection/manager.py,sha256=k8VRqmlfWGKj1IuijbG49jXkMelE8v59pTQTCwkSMKk,38833
+valor_lite/object_detection/metric.py,sha256=nWSqIQSBQrpl3Stz_xe2-AYoo2nrATeMuFVFmREjSNA,23833
+valor_lite/semantic_segmentation/__init__.py,sha256=IdarTHKUuUMDvMBmInQu12Mm_NMCbql6Hf0nL5b56Ak,424
+valor_lite/semantic_segmentation/annotation.py,sha256=CujYFdHS3fgr4Y7mEDs_u1XBmbPJzNU2CdqvjCT_d_A,2938
+valor_lite/semantic_segmentation/computation.py,sha256=iJkEmTNmw9HwQCxSnpJkQsAdVcFriGhhu_WMks6D7tU,5122
+valor_lite/semantic_segmentation/manager.py,sha256=aJk6edWZWKqrzl6hVmEUSZVYhHLuyihxWgAIXsCXkZ0,17361
+valor_lite/semantic_segmentation/metric.py,sha256=Y8M3z92SaABEe9TwBUN37TFsh9DR5WoIxO-TfXVwz8I,6289
+valor_lite/text_generation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite-0.33.9.dist-info/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite-0.33.9.dist-info/METADATA,sha256=dXS7Nt_WHKBaIARWZ3Ek27i26-pWyatewb3eFEnYor8,5631
+valor_lite-0.33.9.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+valor_lite-0.33.9.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.33.9.dist-info/RECORD,,

valor_lite/detection/annotation.py

@@ -1,98 +0,0 @@
-from dataclasses import dataclass, field
-
-import numpy as np
-from numpy.typing import NDArray
-from shapely.geometry import Polygon as ShapelyPolygon
-
-
-@dataclass
-class BoundingBox:
-    xmin: float
-    xmax: float
-    ymin: float
-    ymax: float
-    labels: list[tuple[str, str]]
-    scores: list[float] = field(default_factory=list)
-
-    def __post_init__(self):
-        if len(self.scores) > 0 and len(self.labels) != len(self.scores):
-            raise ValueError(
-                "If scores are defined, there must be a 1:1 pairing with labels."
-            )
-
-    @property
-    def extrema(self) -> tuple[float, float, float, float]:
-        return (self.xmin, self.xmax, self.ymin, self.ymax)
-
-
-@dataclass
-class Polygon:
-    shape: ShapelyPolygon
-    labels: list[tuple[str, str]]
-    scores: list[float] = field(default_factory=list)
-
-    def __post_init__(self):
-        if not isinstance(self.shape, ShapelyPolygon):
-            raise TypeError("shape must be of type shapely.geometry.Polygon.")
-        if len(self.scores) > 0 and len(self.labels) != len(self.scores):
-            raise ValueError(
-                "If scores are defined, there must be a 1:1 pairing with labels."
-            )
-
-    def to_box(self) -> BoundingBox | None:
-
-        if self.shape.is_empty:
-            return None
-
-        xmin, ymin, xmax, ymax = self.shape.bounds
-
-        return BoundingBox(
-            xmin=xmin,
-            xmax=xmax,
-            ymin=ymin,
-            ymax=ymax,
-            labels=self.labels,
-            scores=self.scores,
-        )
-
-
-@dataclass
-class Bitmask:
-    mask: NDArray[np.bool_]
-    labels: list[tuple[str, str]]
-    scores: list[float] = field(default_factory=list)
-
-    def __post_init__(self):
-        if len(self.scores) > 0 and len(self.labels) != len(self.scores):
-            raise ValueError(
-                "If scores are defined, there must be a 1:1 pairing with labels."
-            )
-
-    def to_box(self) -> BoundingBox | None:
-
-        if not self.mask.any():
-            return None
-
-        rows, cols = np.nonzero(self.mask)
-        return BoundingBox(
-            xmin=cols.min(),
-            xmax=cols.max(),
-            ymin=rows.min(),
-            ymax=rows.max(),
-            labels=self.labels,
-            scores=self.scores,
-        )
-
-
-@dataclass
-class Detection:
-    uid: str
-    groundtruths: list[BoundingBox] | list[Bitmask] | list[Polygon]
-    predictions: list[BoundingBox] | list[Bitmask] | list[Polygon]
-
-    def __post_init__(self):
-        for prediction in self.predictions:
-            if len(prediction.scores) != len(prediction.labels):
-                raise ValueError(
-                    "Predictions must provide a score for every label."
-                )