valor-lite 0.33.8__tar.gz → 0.33.10__tar.gz

valor_lite-0.33.10/PKG-INFO

@@ -0,0 +1,179 @@
+Metadata-Version: 2.1
+Name: valor-lite
+Version: 0.33.10
+Summary: Compute valor metrics locally.
+License: MIT License
+        
+        Copyright (c) 2023 Striveworks
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: homepage, https://www.striveworks.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: Pillow>=9.1.0
+Requires-Dist: importlib_metadata; python_version < "3.8"
+Requires-Dist: tqdm
+Requires-Dist: requests
+Requires-Dist: numpy
+Requires-Dist: shapely
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: coverage; extra == "test"
+
+# valor-lite: Fast, local machine learning evaluation.
+
+valor-lite is a lightweight, numpy-based library designed for fast and seamless evaluation of machine learning models. It is optimized for environments where quick, responsive evaluations are essential, whether as part of a larger service or embedded within user-facing tools.
+
+valor-lite is maintained by Striveworks, a cutting-edge MLOps company based in Austin, Texas. If you'd like to learn more or have questions, we invite you to connect with us on [Slack](https://striveworks-public.slack.com/join/shared_invite/zt-1a0jx768y-2J1fffN~b4fXYM8GecvOhA#/shared-invite/email) or explore our [GitHub repository](https://github.com/striveworks/valor).
+
+For additional details, be sure to check out our user [documentation](https://striveworks.github.io/valor/). We're excited to support you in making the most of Valor!
+
+## Usage
+
+### Classification
+
+```python
+from valor_lite.classification import DataLoader, Classification, MetricType
+
+classifications = [
+    Classification(
+        uid="uid0",
+        groundtruth="dog",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.75, 0.2, 0.05],
+    ),
+    Classification(
+        uid="uid1",
+        groundtruth="cat",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.41, 0.39, 0.1],
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(classifications)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': [0.5],
+    'parameters': {
+        'score_thresholds': [0.0],
+        'hardmax': True,
+        'label': 'dog'
+    }
+}
+```
+
+### Object Detection
+
+```python
+from valor_lite.object_detection import DataLoader, Detection, BoundingBox, MetricType
+
+detections = [
+    Detection(
+        uid="uid0",
+        groundtruths=[
+            BoundingBox(
+                xmin=0, xmax=10,
+                ymin=0, ymax=10,
+                labels=["dog"]
+            ),
+            BoundingBox(
+                xmin=20, xmax=30,
+                ymin=20, ymax=30,
+                labels=["cat"]
+            ),
+        ],
+        predictions=[
+            BoundingBox(
+                xmin=1, xmax=11,
+                ymin=1, ymax=11,
+                labels=["dog", "cat", "bird"],
+                scores=[0.85, 0.1, 0.05]
+            ),
+            BoundingBox(
+                xmin=21, xmax=31,
+                ymin=21, ymax=31,
+                labels=["dog", "cat", "bird"],
+                scores=[0.34, 0.33, 0.33]
+            ),
+        ],
+    ),
+]
+
+loader = DataLoader()
+loader.add_bounding_boxes(detections)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': 0.5,
+    'parameters': {
+        'iou_threshold': 0.5,
+        'score_threshold': 0.5,
+        'label': 'dog'
+    }
+}
+```
+
+### Semantic Segmentation
+
+```python
+import numpy as np
+from valor_lite.semantic_segmentation import DataLoader, Segmentation, Bitmask, MetricType
+
+segmentations = [
+    Segmentation(
+        uid="uid0",
+        groundtruths=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ],
+        predictions=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ]
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(segmentations)
+evaluator = loader.finalize()
+
+print(metrics[MetricType.Precision][0])
+```

valor_lite-0.33.10/README.md

@@ -0,0 +1,139 @@
+# valor-lite: Fast, local machine learning evaluation.
+
+valor-lite is a lightweight, numpy-based library designed for fast and seamless evaluation of machine learning models. It is optimized for environments where quick, responsive evaluations are essential, whether as part of a larger service or embedded within user-facing tools.
+
+valor-lite is maintained by Striveworks, a cutting-edge MLOps company based in Austin, Texas. If you'd like to learn more or have questions, we invite you to connect with us on [Slack](https://striveworks-public.slack.com/join/shared_invite/zt-1a0jx768y-2J1fffN~b4fXYM8GecvOhA#/shared-invite/email) or explore our [GitHub repository](https://github.com/striveworks/valor).
+
+For additional details, be sure to check out our user [documentation](https://striveworks.github.io/valor/). We're excited to support you in making the most of Valor!
+
+## Usage
+
+### Classification
+
+```python
+from valor_lite.classification import DataLoader, Classification, MetricType
+
+classifications = [
+    Classification(
+        uid="uid0",
+        groundtruth="dog",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.75, 0.2, 0.05],
+    ),
+    Classification(
+        uid="uid1",
+        groundtruth="cat",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.41, 0.39, 0.1],
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(classifications)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': [0.5],
+    'parameters': {
+        'score_thresholds': [0.0],
+        'hardmax': True,
+        'label': 'dog'
+    }
+}
+```
+
+### Object Detection
+
+```python
+from valor_lite.object_detection import DataLoader, Detection, BoundingBox, MetricType
+
+detections = [
+    Detection(
+        uid="uid0",
+        groundtruths=[
+            BoundingBox(
+                xmin=0, xmax=10,
+                ymin=0, ymax=10,
+                labels=["dog"]
+            ),
+            BoundingBox(
+                xmin=20, xmax=30,
+                ymin=20, ymax=30,
+                labels=["cat"]
+            ),
+        ],
+        predictions=[
+            BoundingBox(
+                xmin=1, xmax=11,
+                ymin=1, ymax=11,
+                labels=["dog", "cat", "bird"],
+                scores=[0.85, 0.1, 0.05]
+            ),
+            BoundingBox(
+                xmin=21, xmax=31,
+                ymin=21, ymax=31,
+                labels=["dog", "cat", "bird"],
+                scores=[0.34, 0.33, 0.33]
+            ),
+        ],
+    ),
+]
+
+loader = DataLoader()
+loader.add_bounding_boxes(detections)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': 0.5,
+    'parameters': {
+        'iou_threshold': 0.5,
+        'score_threshold': 0.5,
+        'label': 'dog'
+    }
+}
+```
+
+### Semantic Segmentation
+
+```python
+import numpy as np
+from valor_lite.semantic_segmentation import DataLoader, Segmentation, Bitmask, MetricType
+
+segmentations = [
+    Segmentation(
+        uid="uid0",
+        groundtruths=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ],
+        predictions=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ]
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(segmentations)
+evaluator = loader.finalize()
+
+print(metrics[MetricType.Precision][0])
+```

{valor_lite-0.33.8 → valor_lite-0.33.10}/benchmarks/benchmark_classification.py

@@ -7,7 +7,29 @@ from time import time
 
 import requests
 from tqdm import tqdm
-from valor_lite.classification import DataLoader, MetricType
+from valor_lite.classification import Classification, DataLoader
+
+
+def _convert_valor_dicts_into_Classification(gt_dict: dict, pd_dict: dict):
+    """Convert a groundtruth dictionary and prediction dictionary into a valor_lite Classification object."""
+    pds = []
+    scores = []
+
+    # there's only one annotation / label per groundtruth in the benchmarking data
+    gt = gt_dict["annotations"][0]["labels"][0]["value"]
+    pds = []
+    scores = []
+    for pann in pd_dict["annotations"]:
+        for valor_label in pann["labels"]:
+            pds.append(valor_label["value"])
+            scores.append(valor_label["score"])
+
+    return Classification(
+        uid=gt_dict["datum"]["uid"],
+        groundtruth=gt,
+        predictions=pds,
+        scores=scores,
+    )
 
 
 def time_it(fn):
@@ -76,35 +98,28 @@ def ingest(
     with open(gt_path, "r") as gf:
         with open(pd_path, "r") as pf:
             count = 0
-            groundtruths = []
-            predictions = []
+            classifications = []
             for gline, pline in zip(gf, pf):
 
-                # groundtruth
                 gt_dict = json.loads(gline)
-                groundtruths.append(gt_dict)
-
-                # prediction
                 pd_dict = json.loads(pline)
-                predictions.append(pd_dict)
-
+                classifications.append(
+                    _convert_valor_dicts_into_Classification(
+                        gt_dict=gt_dict, pd_dict=pd_dict
+                    )
+                )
                 count += 1
                 if count >= limit and limit > 0:
                     break
-                elif len(groundtruths) < chunk_size or chunk_size == -1:
+                elif len(classifications) < chunk_size or chunk_size == -1:
                     continue
 
-                timer, _ = time_it(loader.add_data_from_valor_dict)(
-                    zip(groundtruths, predictions), True
-                )
+                timer, _ = time_it(loader.add_data)(classifications)
                 accumulated_time += timer
-                groundtruths = []
-                predictions = []
+                classifications = []
 
-            if groundtruths:
-                timer, _ = time_it(loader.add_data_from_valor_dict)(
-                    zip(groundtruths, predictions), True
-                )
+            if classifications:
+                timer, _ = time_it(loader.add_data)(classifications)
                 accumulated_time += timer
 
     return accumulated_time
@@ -196,14 +211,16 @@ def run_benchmarking_analysis(
             )
 
         # evaluate
-        eval_time, _ = time_it(evaluator.evaluate)()
+        eval_time, _ = time_it(evaluator.compute_precision_recall)()
         if eval_time > evaluation_timeout and evaluation_timeout != -1:
             raise TimeoutError(
                 f"Base evaluation timed out with {evaluator.n_datums} datums."
             )
 
-        detail_no_examples_time, _ = time_it(evaluator.evaluate)(
-            metrics_to_return=[*MetricType.base(), MetricType.ConfusionMatrix],
+        detail_no_examples_time, _ = time_it(
+            evaluator.compute_confusion_matrix
+        )(
+            number_of_examples=0,
         )
         if (
             detail_no_examples_time > evaluation_timeout
@@ -213,8 +230,9 @@ def run_benchmarking_analysis(
                 f"Base evaluation timed out with {evaluator.n_datums} datums."
             )
 
-        detail_three_examples_time, _ = time_it(evaluator.evaluate)(
-            metrics_to_return=[*MetricType.base(), MetricType.ConfusionMatrix],
+        detail_three_examples_time, _ = time_it(
+            evaluator.compute_confusion_matrix
+        )(
             number_of_examples=3,
         )
         if (

{valor_lite-0.33.8 → valor_lite-0.33.10}/benchmarks/benchmark_objdet.py

@@ -8,7 +8,76 @@ from time import time
 
 import requests
 from tqdm import tqdm
-from valor_lite.detection import DataLoader, MetricType
+from valor_lite.object_detection import BoundingBox, DataLoader, Detection
+
+
+def _get_bbox_extrema(
+    data: list,
+) -> tuple[float, float, float, float]:
+    """Get the bounding box coordinates from a valor Annotation object."""
+    x = [point[0] for shape in data for point in shape]
+    y = [point[1] for shape in data for point in shape]
+    return (min(x), max(x), min(y), max(y))
+
+
+def _convert_valor_dicts_into_Detection(gt_dict: dict, pd_dict: dict):
+    """Convert a groundtruth dictionary and prediction dictionary into a valor_lite Detection object."""
+    gts = []
+    pds = []
+
+    for gann in gt_dict["annotations"]:
+        labels = []
+        for valor_label in gann["labels"]:
+            # NOTE: we only include labels where the key is "name"
+            if valor_label["key"] != "name":
+                continue
+
+            labels.append(valor_label["value"])
+
+        # if the annotation doesn't contain any labels that aren't key == 'name', then we skip that annotation
+        if not labels:
+            continue
+
+        x_min, x_max, y_min, y_max = _get_bbox_extrema(gann["bounding_box"])
+
+        gts.append(
+            BoundingBox(
+                xmin=x_min,
+                xmax=x_max,
+                ymin=y_min,
+                ymax=y_max,
+                labels=labels,
+            )
+        )
+
+    for pann in pd_dict["annotations"]:
+        labels, scores = [], []
+        for valor_label in pann["labels"]:
+            if valor_label["key"] != "name":
+                continue
+            labels.append(valor_label["value"])
+            scores.append(valor_label["score"])
+        if not labels:
+            continue
+
+        x_min, x_max, y_min, y_max = _get_bbox_extrema(pann["bounding_box"])
+
+        pds.append(
+            BoundingBox(
+                xmin=x_min,
+                xmax=x_max,
+                ymin=y_min,
+                ymax=y_max,
+                labels=labels,
+                scores=scores,
+            )
+        )
+
+    return Detection(
+        uid=gt_dict["datum"]["uid"],
+        groundtruths=gts,
+        predictions=pds,
+    )
 
 
 class AnnotationType(str, Enum):
@@ -95,35 +164,29 @@ def ingest(
         with open(pd_path, "r") as pf:
 
             count = 0
-            groundtruths = []
-            predictions = []
+            detections = []
             for gline, pline in zip(gf, pf):
 
-                # groundtruth
                 gt_dict = json.loads(gline)
-                groundtruths.append(gt_dict)
-
-                # prediction
                 pd_dict = json.loads(pline)
-                predictions.append(pd_dict)
+                detections.append(
+                    _convert_valor_dicts_into_Detection(
+                        gt_dict=gt_dict, pd_dict=pd_dict
+                    )
+                )
 
                 count += 1
                 if count >= limit and limit > 0:
                     break
-                elif len(groundtruths) < chunk_size or chunk_size == -1:
+                elif len(detections) < chunk_size or chunk_size == -1:
                     continue
 
-                timer, _ = time_it(manager.add_bounding_boxes_from_valor_dict)(
-                    zip(groundtruths, predictions), True
-                )
+                timer, _ = time_it(manager.add_bounding_boxes)(detections)
                 accumulated_time += timer
-                groundtruths = []
-                predictions = []
+                detections = []
 
-            if groundtruths:
-                timer, _ = time_it(manager.add_bounding_boxes_from_valor_dict)(
-                    zip(groundtruths, predictions), True
-                )
+            if detections:
+                timer, _ = time_it(manager.add_bounding_boxes)(detections)
                 accumulated_time += timer
 
     return accumulated_time
@@ -259,7 +322,7 @@ def run_benchmarking_analysis(
                 )
 
             # evaluate - base metrics only
-            eval_time, metrics = time_it(evaluator.evaluate)()
+            eval_time, metrics = time_it(evaluator.compute_precision_recall)()
             if eval_time > evaluation_timeout and evaluation_timeout != -1:
                 raise TimeoutError(
                     f"Base evaluation timed out with {evaluator.n_datums} datums."
@@ -267,12 +330,9 @@ def run_benchmarking_analysis(
 
             # evaluate - base metrics + detailed counts with no samples
             detailed_counts_time_no_samples, metrics = time_it(
-                evaluator.evaluate
+                evaluator.compute_confusion_matrix
             )(
-                [
-                    MetricType.ConfusionMatrix,
-                    *MetricType.base_metrics(),
-                ]
+                number_of_examples=0,
             )
             if (
                 detailed_counts_time_no_samples > evaluation_timeout
@@ -284,12 +344,8 @@ def run_benchmarking_analysis(
 
             # evaluate - base metrics + detailed counts with 3 samples
             detailed_counts_time_three_samples, metrics = time_it(
-                evaluator.evaluate
+                evaluator.compute_confusion_matrix
             )(
-                [
-                    MetricType.ConfusionMatrix,
-                    *MetricType.base_metrics(),
-                ],
                 number_of_examples=3,
             )
             if (