valor-lite 0.33.19__tar.gz → 0.34.0__tar.gz

{valor_lite-0.33.19/valor_lite.egg-info → valor_lite-0.34.0}/PKG-INFO

@@ -1,42 +1,16 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.19
-Summary: Compute valor metrics locally.
-License: MIT License
-        
-        Copyright (c) 2023 Striveworks
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-        
+Version: 0.34.0
+Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: tqdm
+Requires-Dist: shapely
 Requires-Dist: evaluate
-Requires-Dist: importlib_metadata; python_version < "3.8"
 Requires-Dist: nltk
-Requires-Dist: numpy
-Requires-Dist: Pillow>=9.1.0
-Requires-Dist: requests
 Requires-Dist: rouge_score
-Requires-Dist: shapely
-Requires-Dist: tqdm
 Provides-Extra: mistral
 Requires-Dist: mistralai>=1.0; extra == "mistral"
 Provides-Extra: openai
@@ -45,6 +19,15 @@ Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: coverage; extra == "test"
 Requires-Dist: pre-commit; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocstrings; extra == "docs"
+Requires-Dist: mkdocstrings-python; extra == "docs"
+Requires-Dist: mkdocs-include-dir-to-nav; extra == "docs"
+Requires-Dist: mkdocs-swagger-ui-tag; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: valor-lite[docs,mistral,openai,test]; extra == "dev"
 
 # valor-lite: Fast, local machine learning evaluation.
 

{valor_lite-0.33.19 → valor_lite-0.34.0}/pyproject.toml

@@ -1,20 +1,17 @@
 [project]
 name = "valor-lite"
 dynamic = ["version"]
-description = "Compute valor metrics locally."
+description = "Evaluate machine learning models."
 readme = "README.md"
 requires-python = ">=3.10"
 license = { file = "LICENSE" }
 dependencies = [
+    "numpy",
+    "tqdm",
+    "shapely",
     "evaluate",
-    "importlib_metadata; python_version < '3.8'",
     "nltk",
-    "numpy",
-    "Pillow >= 9.1.0",
-    "requests",
     "rouge_score",
-    "shapely",
-    "tqdm",
 ]
 
 [project.urls]
@@ -28,6 +25,15 @@ build-backend = "setuptools.build_meta"
 mistral = ["mistralai >= 1.0"]
 openai = ["openai"]
 test = ["pytest", "coverage", "pre-commit"]
+docs = [
+    "mkdocs",
+    "mkdocs-material",
+    "mkdocstrings",
+    "mkdocstrings-python",
+    "mkdocs-include-dir-to-nav",
+    "mkdocs-swagger-ui-tag",
+]
+dev = ["valor-lite[openai, mistral, test, docs]"]
 
 [tool.black]
 line-length = 79

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/classification/computation.py

@@ -212,7 +212,7 @@ def _count_with_examples(
     data: NDArray[np.float64],
     unique_idx: int | list[int],
     label_idx: int | list[int],
-) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.int32]]:
+) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.intp]]:
     """
     Helper function for counting occurences of unique detailed pairs.
 
@@ -231,7 +231,7 @@ def _count_with_examples(
         Examples drawn from the data input.
     NDArray[np.int32]
         Unique label indices.
-    NDArray[np.int32]
+    NDArray[np.intp]
         Counts for each unique label index.
     """
     unique_rows, indices = np.unique(
@@ -288,12 +288,14 @@ def compute_confusion_matrix(
     n_labels = label_metadata.shape[0]
     n_scores = score_thresholds.shape[0]
 
-    confusion_matrix = -1 * np.ones(
+    confusion_matrix = np.full(
         (n_scores, n_labels, n_labels, 2 * n_examples + 1),
+        fill_value=-1.0,
         dtype=np.float32,
     )
-    unmatched_ground_truths = -1 * np.ones(
+    unmatched_ground_truths = np.full(
         (n_scores, n_labels, n_examples + 1),
+        fill_value=-1,
         dtype=np.int32,
     )
 
@@ -387,4 +389,4 @@ def compute_confusion_matrix(
                         1 : misprd_label_examples.shape[0] + 1,
                     ] = misprd_label_examples[:, 0].flatten()
 
-    return confusion_matrix, unmatched_ground_truths
+    return confusion_matrix, unmatched_ground_truths  # type: ignore[reportReturnType]

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/classification/manager.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
+
 from valor_lite.classification.annotation import Classification
 from valor_lite.classification.computation import (
     compute_confusion_matrix,
@@ -38,7 +39,7 @@ filtered_metrics = evaluator.evaluate(filter_mask=filter_mask)
 
 @dataclass
 class Filter:
-    indices: NDArray[np.int32]
+    indices: NDArray[np.intp]
     label_metadata: NDArray[np.int32]
     n_datums: int
 
@@ -169,8 +170,7 @@ class Evaluator:
         label_metadata_per_datum = self._label_metadata_per_datum.copy()
         label_metadata_per_datum[:, ~mask] = 0
 
-        label_metadata = np.zeros_like(self._label_metadata, dtype=np.int32)
-        label_metadata = np.transpose(
+        label_metadata: NDArray[np.int32] = np.transpose(
             np.sum(
                 label_metadata_per_datum,
                 axis=1,

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/classification/utilities.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 
 import numpy as np
 from numpy.typing import NDArray
+
 from valor_lite.classification.metric import Metric, MetricType
 
 

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/object_detection/computation.py

@@ -381,9 +381,9 @@ def compute_precion_recall(
             _, indices_gt_unique = np.unique(
                 tp_candidates[:, [0, 1, 4]], axis=0, return_index=True
             )
-            mask_gt_unique = np.zeros(tp_candidates.shape[0], dtype=bool)
+            mask_gt_unique = np.zeros(tp_candidates.shape[0], dtype=np.bool_)
             mask_gt_unique[indices_gt_unique] = True
-            true_positives_mask = np.zeros(n_rows, dtype=bool)
+            true_positives_mask = np.zeros(n_rows, dtype=np.bool_)
             true_positives_mask[mask_tp_inner] = mask_gt_unique
 
             # calculate intermediates
@@ -452,9 +452,9 @@ def compute_precion_recall(
         _, indices_gt_unique = np.unique(
             tp_candidates[:, [0, 1, 4]], axis=0, return_index=True
         )
-        mask_gt_unique = np.zeros(tp_candidates.shape[0], dtype=bool)
+        mask_gt_unique = np.zeros(tp_candidates.shape[0], dtype=np.bool_)
         mask_gt_unique[indices_gt_unique] = True
-        true_positives_mask = np.zeros(n_rows, dtype=bool)
+        true_positives_mask = np.zeros(n_rows, dtype=np.bool_)
         true_positives_mask[mask_tp_outer] = mask_gt_unique
 
         # count running tp and total for AP
@@ -501,8 +501,8 @@ def compute_precion_recall(
         )
 
     # calculate average precision
-    running_max_precision = np.zeros((n_ious, n_labels))
-    running_max_score = np.zeros((n_labels))
+    running_max_precision = np.zeros((n_ious, n_labels), dtype=np.float64)
+    running_max_score = np.zeros((n_labels), dtype=np.float64)
     for recall in range(100, -1, -1):
 
         # running max precision
@@ -528,8 +528,12 @@ def compute_precion_recall(
 
     # calculate mAP and mAR
     if unique_pd_labels.size > 0:
-        mAP = average_precision[:, unique_pd_labels].mean(axis=1)
-        mAR = average_recall[:, unique_pd_labels].mean(axis=1)
+        mAP: NDArray[np.float64] = average_precision[:, unique_pd_labels].mean(
+            axis=1
+        )
+        mAR: NDArray[np.float64] = average_recall[:, unique_pd_labels].mean(
+            axis=1
+        )
     else:
         mAP = np.zeros(n_ious, dtype=np.float64)
         mAR = np.zeros(n_scores, dtype=np.float64)
@@ -561,14 +565,14 @@ def compute_precion_recall(
         accuracy,
         counts,
         pr_curve,
-    )
+    )  # type: ignore[reportReturnType]
 
 
 def _count_with_examples(
     data: NDArray[np.float64],
     unique_idx: int | list[int],
     label_idx: int | list[int],
-) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.int32]]:
+) -> tuple[NDArray[np.float64], NDArray[np.int32], NDArray[np.intp]]:
     """
     Helper function for counting occurences of unique detailed pairs.
 
@@ -587,7 +591,7 @@ def _count_with_examples(
         Examples drawn from the data input.
     NDArray[np.int32]
         Unique label indices.
-    NDArray[np.int32]
+    NDArray[np.intp]
         Counts for each unique label index.
     """
     unique_rows, indices = np.unique(
@@ -907,4 +911,4 @@ def compute_confusion_matrix(
         confusion_matrix,
         unmatched_predictions,
         unmatched_ground_truths,
-    )
+    )  # type: ignore[reportReturnType]

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/object_detection/manager.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
+
 from valor_lite.object_detection.annotation import Detection
 from valor_lite.object_detection.computation import (
     compute_bbox_iou,
@@ -42,8 +43,8 @@ filtered_metrics = evaluator.evaluate(iou_thresholds=[0.5], filter_mask=filter_m
 
 @dataclass
 class Filter:
-    ranked_indices: NDArray[np.int32]
-    detailed_indices: NDArray[np.int32]
+    ranked_indices: NDArray[np.intp]
+    detailed_indices: NDArray[np.intp]
     label_metadata: NDArray[np.int32]
 
 
@@ -569,7 +570,8 @@ class DataLoader:
                         [gt.extrema, pd.extrema]
                         for pd in detection.predictions
                         for gt in detection.groundtruths
-                    ]
+                    ],
+                    dtype=np.float64,
                 )
             ).reshape(len(detection.predictions), len(detection.groundtruths))
             for detection in detections

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/object_detection/utilities.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 
 import numpy as np
 from numpy.typing import NDArray
+
 from valor_lite.object_detection.metric import Metric, MetricType
 
 
@@ -136,10 +137,8 @@ def unpack_precision_recall_into_metric_lists(
 
     metrics[MetricType.PrecisionRecallCurve] = [
         Metric.precision_recall_curve(
-            precisions=pr_curves[iou_idx, label_idx, :, 0]
-            .astype(float)
-            .tolist(),
-            scores=pr_curves[iou_idx, label_idx, :, 1].astype(float).tolist(),
+            precisions=pr_curves[iou_idx, label_idx, :, 0].tolist(),  # type: ignore[reportArgumentType]
+            scores=pr_curves[iou_idx, label_idx, :, 1].tolist(),  # type: ignore[reportArgumentType]
             iou_threshold=iou_threshold,
             label=label,
         )

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/semantic_segmentation/manager.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 import numpy as np
 from numpy.typing import NDArray
 from tqdm import tqdm
+
 from valor_lite.semantic_segmentation.annotation import Segmentation
 from valor_lite.semantic_segmentation.computation import (
     compute_intermediate_confusion_matrices,
@@ -37,7 +38,7 @@ filtered_metrics = evaluator.evaluate(filter_mask=filter_mask)
 
 @dataclass
 class Filter:
-    indices: NDArray[np.int32]
+    indices: NDArray[np.intp]
     label_metadata: NDArray[np.int32]
     n_pixels: int
 

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/semantic_segmentation/utilities.py

@@ -2,6 +2,7 @@ from collections import defaultdict
 
 import numpy as np
 from numpy.typing import NDArray
+
 from valor_lite.semantic_segmentation.metric import Metric, MetricType
 
 

{valor_lite-0.33.19 → valor_lite-0.34.0}/valor_lite/text_generation/computation.py

@@ -1,6 +1,7 @@
 import evaluate
 from nltk.tokenize import RegexpTokenizer
 from nltk.translate import bleu_score
+
 from valor_lite.text_generation.llm.generation import (
     generate_answer_correctness_verdicts,
     generate_answer_relevance_verdicts,

{valor_lite-0.33.19 → valor_lite-0.34.0/valor_lite.egg-info}/PKG-INFO

@@ -1,42 +1,16 @@
 Metadata-Version: 2.1
 Name: valor-lite
-Version: 0.33.19
-Summary: Compute valor metrics locally.
-License: MIT License
-        
-        Copyright (c) 2023 Striveworks
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
-        
+Version: 0.34.0
+Summary: Evaluate machine learning models.
 Project-URL: homepage, https://www.striveworks.com
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: tqdm
+Requires-Dist: shapely
 Requires-Dist: evaluate
-Requires-Dist: importlib_metadata; python_version < "3.8"
 Requires-Dist: nltk
-Requires-Dist: numpy
-Requires-Dist: Pillow>=9.1.0
-Requires-Dist: requests
 Requires-Dist: rouge_score
-Requires-Dist: shapely
-Requires-Dist: tqdm
 Provides-Extra: mistral
 Requires-Dist: mistralai>=1.0; extra == "mistral"
 Provides-Extra: openai
@@ -45,6 +19,15 @@ Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: coverage; extra == "test"
 Requires-Dist: pre-commit; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocstrings; extra == "docs"
+Requires-Dist: mkdocstrings-python; extra == "docs"
+Requires-Dist: mkdocs-include-dir-to-nav; extra == "docs"
+Requires-Dist: mkdocs-swagger-ui-tag; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: valor-lite[docs,mistral,openai,test]; extra == "dev"
 
 # valor-lite: Fast, local machine learning evaluation.
 

valor_lite-0.34.0/valor_lite.egg-info/SOURCES.txt

@@ -0,0 +1,42 @@
+README.md
+pyproject.toml
+valor_lite/LICENSE
+valor_lite/__init__.py
+valor_lite/profiling.py
+valor_lite/schemas.py
+valor_lite.egg-info/PKG-INFO
+valor_lite.egg-info/SOURCES.txt
+valor_lite.egg-info/dependency_links.txt
+valor_lite.egg-info/requires.txt
+valor_lite.egg-info/top_level.txt
+valor_lite/classification/__init__.py
+valor_lite/classification/annotation.py
+valor_lite/classification/computation.py
+valor_lite/classification/manager.py
+valor_lite/classification/metric.py
+valor_lite/classification/utilities.py
+valor_lite/object_detection/__init__.py
+valor_lite/object_detection/annotation.py
+valor_lite/object_detection/computation.py
+valor_lite/object_detection/manager.py
+valor_lite/object_detection/metric.py
+valor_lite/object_detection/utilities.py
+valor_lite/semantic_segmentation/__init__.py
+valor_lite/semantic_segmentation/annotation.py
+valor_lite/semantic_segmentation/benchmark.py
+valor_lite/semantic_segmentation/computation.py
+valor_lite/semantic_segmentation/manager.py
+valor_lite/semantic_segmentation/metric.py
+valor_lite/semantic_segmentation/utilities.py
+valor_lite/text_generation/__init__.py
+valor_lite/text_generation/annotation.py
+valor_lite/text_generation/computation.py
+valor_lite/text_generation/manager.py
+valor_lite/text_generation/metric.py
+valor_lite/text_generation/llm/__init__.py
+valor_lite/text_generation/llm/exceptions.py
+valor_lite/text_generation/llm/generation.py
+valor_lite/text_generation/llm/instructions.py
+valor_lite/text_generation/llm/integrations.py
+valor_lite/text_generation/llm/utilities.py
+valor_lite/text_generation/llm/validators.py

valor_lite-0.34.0/valor_lite.egg-info/requires.txt

@@ -0,0 +1,28 @@
+numpy
+tqdm
+shapely
+evaluate
+nltk
+rouge_score
+
+[dev]
+valor-lite[docs,mistral,openai,test]
+
+[docs]
+mkdocs
+mkdocs-material
+mkdocstrings
+mkdocstrings-python
+mkdocs-include-dir-to-nav
+mkdocs-swagger-ui-tag
+
+[mistral]
+mistralai>=1.0
+
+[openai]
+openai
+
+[test]
+pytest
+coverage
+pre-commit

valor_lite-0.33.19/benchmarks/.gitignore

@@ -1,2 +0,0 @@
-*.json
-*.jsonl