valor-lite 0.37.1__py3-none-any.whl

valor_lite/text_generation/metric.py

@@ -0,0 +1,381 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from valor_lite.schemas import BaseMetric
+
+
+class MetricType(str, Enum):
+    AnswerCorrectness = "AnswerCorrectness"
+    AnswerRelevance = "AnswerRelevance"
+    Bias = "Bias"
+    BLEU = "BLEU"
+    ContextPrecision = "ContextPrecision"
+    ContextRecall = "ContextRecall"
+    ContextRelevance = "ContextRelevance"
+    Faithfulness = "Faithfulness"
+    Hallucination = "Hallucination"
+    ROUGE = "ROUGE"
+    SummaryCoherence = "SummaryCoherence"
+    Toxicity = "Toxicity"
+
+
+@dataclass
+class Metric(BaseMetric):
+    """
+    Text Generation Metric.
+
+    Attributes
+    ----------
+    type : str
+        The metric type.
+    value : int | float | dict
+        The metric value.
+    parameters : dict[str, Any]
+        A dictionary containing metric parameters.
+    """
+
+    def __post_init__(self):
+        if not isinstance(self.type, str):
+            raise TypeError(
+                f"Metric type should be of type 'str': {self.type}"
+            )
+        elif not isinstance(self.value, (int, float, dict)):
+            raise TypeError(
+                f"Metric value must be of type 'int', 'float' or 'dict': {self.value}"
+            )
+        elif not isinstance(self.parameters, dict):
+            raise TypeError(
+                f"Metric parameters must be of type 'dict[str, Any]': {self.parameters}"
+            )
+        elif not all([isinstance(k, str) for k in self.parameters.keys()]):
+            raise TypeError(
+                f"Metric parameter dictionary should only have keys with type 'str': {self.parameters}"
+            )
+
+    @classmethod
+    def error(
+        cls,
+        error_type: str,
+        error_message: str,
+        model_name: str,
+        retries: int,
+    ):
+        return cls(
+            type="Error",
+            value={
+                "type": error_type,
+                "message": error_message,
+            },
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def answer_correctness(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines an answer correctness metric.
+
+        Parameters
+        ----------
+        value : float
+            The answer correctness score between 0 and 1, with higher values indicating that the answer
+            is more correct. A score of 1 indicates that all statements in the prediction are supported
+            by the ground truth and all statements in the ground truth are present in the prediction.
+        """
+        return cls(
+            type=MetricType.AnswerCorrectness,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def answer_relevance(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines an answer relevance metric.
+
+        Parameters
+        ----------
+        value : float
+            The number of statements in the answer that are relevant to the query divided by the total
+            number of statements in the answer.
+        """
+        return cls(
+            type=MetricType.AnswerRelevance,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def bleu(
+        cls,
+        value: float,
+        weights: list[float],
+    ):
+        """
+        Defines a BLEU metric.
+
+        Parameters
+        ----------
+        value : float
+            The BLEU score for an individual datapoint.
+        weights : list[float]
+            The list of weights that the score was calculated with.
+        """
+        return cls(
+            type=MetricType.BLEU,
+            value=value,
+            parameters={
+                "weights": weights,
+            },
+        )
+
+    @classmethod
+    def bias(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines a bias metric.
+
+        Parameters
+        ----------
+        value : float
+            The bias score for a datum. This is a float between 0 and 1, with 1 indicating that all
+            opinions in the datum text are biased and 0 indicating that there is no bias.
+        """
+        return cls(
+            type=MetricType.Bias,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def context_precision(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines a context precision metric.
+
+        Parameters
+        ----------
+        value : float
+            The context precision score for a datum. This is a float between 0 and 1, with 0 indicating
+            that none of the contexts are useful to arrive at the ground truth answer to the query
+            and 1 indicating that all contexts are useful to arrive at the ground truth answer to the
+            query. The score is more heavily influenced by earlier contexts in the list of contexts
+            than later contexts.
+        """
+        return cls(
+            type=MetricType.ContextPrecision,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def context_recall(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines a context recall metric.
+
+        Parameters
+        ----------
+        value : float
+            The context recall score for a datum. This is a float between 0 and 1, with 1 indicating
+            that all ground truth statements are attributable to the context list.
+        """
+        return cls(
+            type=MetricType.ContextRecall,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def context_relevance(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines a context relevance metric.
+
+        Parameters
+        ----------
+        value : float
+            The context relevance score for a datum. This is a float between 0 and 1, with 0 indicating
+            that none of the contexts are relevant and 1 indicating that all of the contexts are relevant.
+        """
+        return cls(
+            type=MetricType.ContextRelevance,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def faithfulness(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines a faithfulness metric.
+
+        Parameters
+        ----------
+        value : float
+            The faithfulness score for a datum. This is a float between 0 and 1, with 1 indicating that
+            all claims in the text are implied by the contexts.
+        """
+        return cls(
+            type=MetricType.Faithfulness,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def hallucination(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines a hallucination metric.
+
+        Parameters
+        ----------
+        value : float
+            The hallucination score for a datum. This is a float between 0 and 1, with 1 indicating that
+            all contexts are contradicted by the text.
+        """
+        return cls(
+            type=MetricType.Hallucination,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def rouge(
+        cls,
+        value: float,
+        rouge_type: str,
+        use_stemmer: bool,
+    ):
+        """
+        Defines a ROUGE metric.
+
+        Parameters
+        ----------
+        value : float
+            A ROUGE score.
+        rouge_type : ROUGEType
+            The ROUGE variation used to compute the value. `rouge1` is unigram-based scoring, `rouge2` is bigram-based
+            scoring, `rougeL` is scoring based on sentences (i.e., splitting on "." and ignoring "\n"), and `rougeLsum`
+            is scoring based on splitting the text using "\n".
+        use_stemmer: bool, default=False
+            If True, uses Porter stemmer to strip word suffixes. Defaults to False.
+        """
+        return cls(
+            type=MetricType.ROUGE,
+            value=value,
+            parameters={
+                "rouge_type": rouge_type,
+                "use_stemmer": use_stemmer,
+            },
+        )
+
+    @classmethod
+    def summary_coherence(
+        cls,
+        value: int,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines a summary coherence metric.
+
+        Parameters
+        ----------
+        value : int
+            The summary coherence score for a datum. This is an integer with 1 being the lowest summary coherence
+            and 5 the highest summary coherence.
+        """
+        return cls(
+            type=MetricType.SummaryCoherence,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )
+
+    @classmethod
+    def toxicity(
+        cls,
+        value: float,
+        model_name: str,
+        retries: int,
+    ):
+        """
+        Defines a toxicity metric.
+
+        Parameters
+        ----------
+        value : float
+            The toxicity score for a datum. This is a value between 0 and 1, with 1 indicating that all opinions
+            in the datum text are toxic and 0 indicating that there is no toxicity.
+        """
+        return cls(
+            type=MetricType.Toxicity,
+            value=value,
+            parameters={
+                "evaluator": model_name,
+                "retries": retries,
+            },
+        )

valor_lite-0.37.1.dist-info/METADATA

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: valor-lite
+Version: 0.37.1
+Summary: Evaluate machine learning models.
+Project-URL: homepage, https://www.striveworks.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: tqdm
+Requires-Dist: shapely
+Requires-Dist: pyarrow
+Provides-Extra: nlp
+Requires-Dist: evaluate; extra == "nlp"
+Requires-Dist: nltk; extra == "nlp"
+Requires-Dist: rouge_score; extra == "nlp"
+Provides-Extra: mistral
+Requires-Dist: mistralai<1.8.0,>=1.0.0; extra == "mistral"
+Provides-Extra: openai
+Requires-Dist: openai; extra == "openai"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocstrings; extra == "docs"
+Requires-Dist: mkdocstrings-python; extra == "docs"
+Requires-Dist: mkdocs-include-dir-to-nav; extra == "docs"
+Requires-Dist: mkdocs-swagger-ui-tag; extra == "docs"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: coverage; extra == "test"
+Requires-Dist: pre-commit; extra == "test"
+Provides-Extra: benchmark
+Requires-Dist: requests; extra == "benchmark"
+Provides-Extra: dev
+Requires-Dist: valor-lite[benchmark,docs,mistral,nlp,openai,test]; extra == "dev"
+
+# valor-lite: Fast, local machine learning evaluation.
+
+valor-lite is a lightweight, numpy-based library designed for fast and seamless evaluation of machine learning models. It is optimized for environments where quick, responsive evaluations are essential, whether as part of a larger service or embedded within user-facing tools.
+
+valor-lite is maintained by Striveworks, a cutting-edge MLOps company based in Austin, Texas. If you'd like to learn more or have questions, we invite you to connect with us on [Slack](https://striveworks-public.slack.com/join/shared_invite/zt-1a0jx768y-2J1fffN~b4fXYM8GecvOhA#/shared-invite/email) or explore our [GitHub repository](https://github.com/striveworks/valor).
+
+For additional details, be sure to check out our user [documentation](https://striveworks.github.io/valor/). We're excited to support you in making the most of Valor!
+
+## Usage
+
+### Classification
+
+```python
+from valor_lite.classification import DataLoader, Classification, MetricType
+
+classifications = [
+    Classification(
+        uid="uid0",
+        groundtruth="dog",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.75, 0.2, 0.05],
+    ),
+    Classification(
+        uid="uid1",
+        groundtruth="cat",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.41, 0.39, 0.1],
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(classifications)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': [0.5],
+    'parameters': {
+        'score_thresholds': [0.0],
+        'hardmax': True,
+        'label': 'dog'
+    }
+}
+```
+
+### Object Detection
+
+```python
+from valor_lite.object_detection import DataLoader, Detection, BoundingBox, MetricType
+
+detections = [
+    Detection(
+        uid="uid0",
+        groundtruths=[
+            BoundingBox(
+                xmin=0, xmax=10,
+                ymin=0, ymax=10,
+                labels=["dog"]
+            ),
+            BoundingBox(
+                xmin=20, xmax=30,
+                ymin=20, ymax=30,
+                labels=["cat"]
+            ),
+        ],
+        predictions=[
+            BoundingBox(
+                xmin=1, xmax=11,
+                ymin=1, ymax=11,
+                labels=["dog", "cat", "bird"],
+                scores=[0.85, 0.1, 0.05]
+            ),
+            BoundingBox(
+                xmin=21, xmax=31,
+                ymin=21, ymax=31,
+                labels=["dog", "cat", "bird"],
+                scores=[0.34, 0.33, 0.33]
+            ),
+        ],
+    ),
+]
+
+loader = DataLoader()
+loader.add_bounding_boxes(detections)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': 0.5,
+    'parameters': {
+        'iou_threshold': 0.5,
+        'score_threshold': 0.5,
+        'label': 'dog'
+    }
+}
+```
+
+### Semantic Segmentation
+
+```python
+import numpy as np
+from valor_lite.semantic_segmentation import DataLoader, Segmentation, Bitmask, MetricType
+
+segmentations = [
+    Segmentation(
+        uid="uid0",
+        groundtruths=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ],
+        predictions=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ]
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(segmentations)
+evaluator = loader.finalize()
+
+print(metrics[MetricType.Precision][0])
+```

valor_lite-0.37.1.dist-info/RECORD

@@ -0,0 +1,49 @@
+valor_lite/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
+valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/exceptions.py,sha256=Odg4m7VfTYtGunzA-JpNWGoDwvXFvCamkQu8WLncE-A,496
+valor_lite/schemas.py,sha256=pB0MrPx5qFLbwBWDiOUUm-vmXdWvbJLFCBmKgbcbI5g,198
+valor_lite/cache/__init__.py,sha256=OzVJq6WMXOmhJPQLOVu9y0xEui41PDpwGnkrU6A3-dA,266
+valor_lite/cache/compute.py,sha256=WR1qPMx7MtdhQSrYbAI9LIHkaEDuzX3na2Cv_aO0W1A,5183
+valor_lite/cache/ephemeral.py,sha256=4TT7ruk8GG0DxIjAFcNEBkpJ5sIdky2eLTS7NwAO758,8149
+valor_lite/cache/persistent.py,sha256=OMXrhfYuleTsCaG1e5aue9Juxaz6kDFVinlCqOX5lPA,15023
+valor_lite/classification/__init__.py,sha256=5cAK1exDJGGhZWFcswj977-qVtEafUTsNIg7stJV2oc,293
+valor_lite/classification/annotation.py,sha256=93NzpfpBhkyLq33gLtAqkCl0TTmraDGy03cSjAxMWec,1304
+valor_lite/classification/computation.py,sha256=x2IINIDHP_52GTCdkTgu8ujx3o3HGosdWE0_oQvNZuU,11498
+valor_lite/classification/evaluator.py,sha256=eoT4_FfR76dUlL-AXqjx7boHA9oCfSAmHwSOIhkW_ug,28855
+valor_lite/classification/loader.py,sha256=t976u0XGBuQGrG6VgxPViu-Ipbj5sUm_ShW8FV0YGxg,3355
+valor_lite/classification/metric.py,sha256=4ZoP9f36DqKnDOif40kjwznJc0fw93F1yx3gvX_lnz8,16104
+valor_lite/classification/numpy_compatibility.py,sha256=roqtTetsm1_HxuaejrthQdydjsRIy-FpXpGb86cLh_E,365
+valor_lite/classification/shared.py,sha256=J3lrcJQrSSEXc1qv7s9CIU3UF_XEZxd6flki1I0nhfA,5653
+valor_lite/classification/utilities.py,sha256=NZP830x0noKSZCC6-Hc_K34JzxLsHgU8Zcq0YjTlH90,10218
+valor_lite/object_detection/__init__.py,sha256=GARztCi3rYlq3gltJ0lDBQK0bVYr5Upwc_M3_Cl_RMg,363
+valor_lite/object_detection/annotation.py,sha256=--uvHOS7vPIz3cqArp8dZM4Ax9_lzW0HCaz626ZULDM,7684
+valor_lite/object_detection/computation.py,sha256=EREV5OokVez70eGNbbuzrJo-PXANKh0XrUqpF8PKHPU,25505
+valor_lite/object_detection/evaluator.py,sha256=2ytNGsizQfnK9ehHE7R1QNxYSzPIRQV4fmmvtMP3FMM,27780
+valor_lite/object_detection/loader.py,sha256=kPGG8UXCBDZl10qfDv3YqailgdHk3O4DyZQ8pqkVhJc,11428
+valor_lite/object_detection/metric.py,sha256=-bcupJvS_BSWNPN2trkO8EWD-z7TDFGGim7fIHpzPzw,27924
+valor_lite/object_detection/shared.py,sha256=y9sNZIKtLLVVGec8aPKH_nuVf-pXXt05BhTJG-WP0yY,5672
+valor_lite/object_detection/utilities.py,sha256=zoez6MaBx0IwxJf-zDQMkCh80lQf0zB4Fl1xluFTtnY,14526
+valor_lite/semantic_segmentation/__init__.py,sha256=OeAKuANM2mvw3JX4pi-eudc82YMqsXJwK1DIjgl2oeI,318
+valor_lite/semantic_segmentation/annotation.py,sha256=XB54BcBu_soQvbP3DrbXCruw-sypJBC6KhLqRWX1Vmw,4384
+valor_lite/semantic_segmentation/computation.py,sha256=dhbwybDe5kQGXzUSOjm15UGjDoGGt5zPtvPyvp8kFc4,4690
+valor_lite/semantic_segmentation/evaluator.py,sha256=jXFBPv_pmIOj85wJFFikbNNWDq8wPG2rcLiaWrgDcUs,13569
+valor_lite/semantic_segmentation/loader.py,sha256=rDtbeGhQEDhGjcjx9RHiSATQM25uftF13c1pm09axUM,7582
+valor_lite/semantic_segmentation/metric.py,sha256=T9RfPJf4WgqGQTXYvSy08vJG5bjXXJnyYZeW0mlxMa8,7132
+valor_lite/semantic_segmentation/shared.py,sha256=i9F7nAoH9Yhabqj-SVtnQfm2ST0xkVFUcQVBVLabPeE,4539
+valor_lite/semantic_segmentation/utilities.py,sha256=MjqSlS1wy0RwJGiuUpZrT3AUMXEahO7-lh2oeNeqGV8,2470
+valor_lite/text_generation/__init__.py,sha256=pGhpWCSZjLM0pPHCtPykAfos55B8ie3mi9EzbNxfj-U,356
+valor_lite/text_generation/annotation.py,sha256=O5aXiwCS4WjA-fqn4ly-O0MsTHoIOmqxqCaAp9IeI3M,1270
+valor_lite/text_generation/computation.py,sha256=hGDkPfzWY9SDTdozd-nArexJ3ZSNlCIWqHGoD8vO2Cc,18652
+valor_lite/text_generation/manager.py,sha256=C4QwvronGHXmYSkaRmUGy7TN0C0aeyDx9Hb-ClNYXK4,24810
+valor_lite/text_generation/metric.py,sha256=C9gbWejjOJ23JVLecuUhYW5rkx30NUCfRtgsM46uMds,10409
+valor_lite/text_generation/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+valor_lite/text_generation/llm/exceptions.py,sha256=w4eYSJIJQ_jWuCmquCB6ETr_st_LzbDRlhnlPeqwmfo,349
+valor_lite/text_generation/llm/generation.py,sha256=XKPjCxPUZHiWInQSO7wLOb0YtMFLu50s8rHZe1Yz0s0,28954
+valor_lite/text_generation/llm/instructions.py,sha256=fz2onBZZWcl5W8iy7zEWkPGU9N07ez6O7SxZA5M2xe4,34056
+valor_lite/text_generation/llm/integrations.py,sha256=-rTfdAjq1zH-4ixwYuMQEOQ80pIFzMTe0BYfroVx3Pg,6974
+valor_lite/text_generation/llm/utilities.py,sha256=bjqatGgtVTcl1PrMwiDKTYPGJXKrBrx7PDtzIblGSys,1178
+valor_lite/text_generation/llm/validators.py,sha256=Wzr5RlfF58_2wOU-uTw7C8skan_fYdhy4Gfn0jSJ8HM,2700
+valor_lite-0.37.1.dist-info/METADATA,sha256=y8qWyMI-Q0GKjbxePQJSuB9sCP4vsVlnECghZb-8YfI,5094
+valor_lite-0.37.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+valor_lite-0.37.1.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
+valor_lite-0.37.1.dist-info/RECORD,,

valor_lite-0.37.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

valor_lite-0.37.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+valor_lite