valor-lite 0.36.6__py3-none-any.whl → 0.37.5__py3-none-any.whl

valor_lite/semantic_segmentation/manager.py

@@ -1,446 +0,0 @@
-from dataclasses import asdict, dataclass
-
-import numpy as np
-from numpy.typing import NDArray
-from tqdm import tqdm
-
-from valor_lite.exceptions import EmptyEvaluatorError, EmptyFilterError
-from valor_lite.semantic_segmentation.annotation import Segmentation
-from valor_lite.semantic_segmentation.computation import (
-    compute_intermediate_confusion_matrices,
-    compute_label_metadata,
-    compute_metrics,
-    filter_cache,
-)
-from valor_lite.semantic_segmentation.metric import Metric, MetricType
-from valor_lite.semantic_segmentation.utilities import (
-    unpack_precision_recall_iou_into_metric_lists,
-)
-
-"""
-Usage
------
-
-manager = DataLoader()
-manager.add_data(
-    groundtruths=groundtruths,
-    predictions=predictions,
-)
-evaluator = manager.finalize()
-
-metrics = evaluator.evaluate()
-
-f1_metrics = metrics[MetricType.F1]
-accuracy_metrics = metrics[MetricType.Accuracy]
-
-filter_mask = evaluator.create_filter(datum_ids=["uid1", "uid2"])
-filtered_metrics = evaluator.evaluate(filter_mask=filter_mask)
-"""
-
-
-@dataclass
-class Metadata:
-    number_of_labels: int = 0
-    number_of_pixels: int = 0
-    number_of_datums: int = 0
-    number_of_ground_truths: int = 0
-    number_of_predictions: int = 0
-
-    @classmethod
-    def create(
-        cls,
-        confusion_matrices: NDArray[np.int64],
-    ):
-        if confusion_matrices.size == 0:
-            return cls()
-        return cls(
-            number_of_labels=confusion_matrices.shape[1] - 1,
-            number_of_pixels=confusion_matrices.sum(),
-            number_of_datums=confusion_matrices.shape[0],
-            number_of_ground_truths=confusion_matrices[:, 1:, :].sum(),
-            number_of_predictions=confusion_matrices[:, :, 1:].sum(),
-        )
-
-    def to_dict(self) -> dict[str, int | bool]:
-        return asdict(self)
-
-
-@dataclass
-class Filter:
-    datum_mask: NDArray[np.bool_]
-    label_mask: NDArray[np.bool_]
-    metadata: Metadata
-
-    def __post_init__(self):
-        # validate datum mask
-        if not self.datum_mask.any():
-            raise EmptyFilterError("filter removes all datums")
-
-        # validate label mask
-        if self.label_mask.all():
-            raise EmptyFilterError("filter removes all labels")
-
-
-class Evaluator:
-    """
-    Segmentation Evaluator
-    """
-
-    def __init__(self):
-        """Initializes evaluator caches."""
-        # external references
-        self.datum_id_to_index: dict[str, int] = {}
-        self.index_to_datum_id: list[str] = []
-        self.label_to_index: dict[str, int] = {}
-        self.index_to_label: list[str] = []
-
-        # internal caches
-        self._confusion_matrices = np.array([], dtype=np.int64)
-        self._label_metadata = np.array([], dtype=np.int64)
-        self._metadata = Metadata()
-
-    @property
-    def metadata(self) -> Metadata:
-        return self._metadata
-
-    @property
-    def ignored_prediction_labels(self) -> list[str]:
-        """
-        Prediction labels that are not present in the ground truth set.
-        """
-        glabels = set(np.where(self._label_metadata[:, 0] > 0)[0])
-        plabels = set(np.where(self._label_metadata[:, 1] > 0)[0])
-        return [
-            self.index_to_label[label_id] for label_id in (plabels - glabels)
-        ]
-
-    @property
-    def missing_prediction_labels(self) -> list[str]:
-        """
-        Ground truth labels that are not present in the prediction set.
-        """
-        glabels = set(np.where(self._label_metadata[:, 0] > 0)[0])
-        plabels = set(np.where(self._label_metadata[:, 1] > 0)[0])
-        return [
-            self.index_to_label[label_id] for label_id in (glabels - plabels)
-        ]
-
-    def create_filter(
-        self,
-        datums: list[str] | NDArray[np.int64] | None = None,
-        labels: list[str] | NDArray[np.int64] | None = None,
-    ) -> Filter:
-        """
-        Creates a filter for use with the evaluator.
-
-        Parameters
-        ----------
-        datums : list[str] | NDArray[int64], optional
-            An optional list of string ids or array of indices representing datums.
-        labels : list[str] | NDArray[int64], optional
-            An optional list of labels or array of indices.
-
-        Returns
-        -------
-        Filter
-            The filter object containing a mask and metadata.
-        """
-        datum_mask = np.ones(self._confusion_matrices.shape[0], dtype=np.bool_)
-        label_mask = np.zeros(
-            self.metadata.number_of_labels + 1, dtype=np.bool_
-        )
-
-        if datums is not None:
-            # convert to indices
-            if isinstance(datums, list):
-                datums = np.array(
-                    [self.datum_id_to_index[uid] for uid in datums],
-                    dtype=np.int64,
-                )
-
-            # validate indices
-            if datums.size == 0:
-                raise EmptyFilterError(
-                    "filter removes all datums"
-                )  # validate indices
-            elif datums.min() < 0:
-                raise ValueError(
-                    f"datum index cannot be negative '{datums.min()}'"
-                )
-            elif datums.max() >= len(self.index_to_datum_id):
-                raise ValueError(
-                    f"datum index cannot exceed total number of datums '{datums.max()}'"
-                )
-
-            # apply to mask
-            datums.sort()
-            mask_valid_datums = (
-                np.arange(self._confusion_matrices.shape[0]).reshape(-1, 1)
-                == datums.reshape(1, -1)
-            ).any(axis=1)
-            datum_mask[~mask_valid_datums] = False
-
-        if labels is not None:
-            # convert to indices
-            if isinstance(labels, list):
-                labels = np.array(
-                    [self.label_to_index[label] for label in labels],
-                    dtype=np.int64,
-                )
-
-            # validate indices
-            if labels.size == 0:
-                raise EmptyFilterError("filter removes all labels")
-            elif labels.min() < 0:
-                raise ValueError(
-                    f"label index cannot be negative '{labels.min()}'"
-                )
-            elif labels.max() >= len(self.index_to_label):
-                raise ValueError(
-                    f"label index cannot exceed total number of labels '{labels.max()}'"
-                )
-
-            # apply to mask
-            labels = np.concatenate([labels, np.array([-1])])
-            label_range = np.arange(self.metadata.number_of_labels + 1) - 1
-            mask_valid_labels = (
-                label_range.reshape(-1, 1) == labels.reshape(1, -1)
-            ).any(axis=1)
-            label_mask[~mask_valid_labels] = True
-
-        filtered_confusion_matrices, _ = filter_cache(
-            confusion_matrices=self._confusion_matrices.copy(),
-            datum_mask=datum_mask,
-            label_mask=label_mask,
-            number_of_labels=self.metadata.number_of_labels,
-        )
-
-        return Filter(
-            datum_mask=datum_mask,
-            label_mask=label_mask,
-            metadata=Metadata.create(
-                confusion_matrices=filtered_confusion_matrices,
-            ),
-        )
-
-    def filter(
-        self, filter_: Filter
-    ) -> tuple[NDArray[np.int64], NDArray[np.int64]]:
-        """
-        Performs the filter operation over the internal cache.
-
-        Parameters
-        ----------
-        filter_ : Filter
-            An object describing the filter operation.
-
-        Returns
-        -------
-        NDArray[int64]
-            Filtered confusion matrices.
-        NDArray[int64]
-            Filtered label metadata
-        """
-        return filter_cache(
-            confusion_matrices=self._confusion_matrices.copy(),
-            datum_mask=filter_.datum_mask,
-            label_mask=filter_.label_mask,
-            number_of_labels=self.metadata.number_of_labels,
-        )
-
-    def compute_precision_recall_iou(
-        self, filter_: Filter | None = None
-    ) -> dict[MetricType, list]:
-        """
-        Performs an evaluation and returns metrics.
-
-        Returns
-        -------
-        dict[MetricType, list]
-            A dictionary mapping MetricType enumerations to lists of computed metrics.
-        """
-        if filter_ is not None:
-            confusion_matrices, label_metadata = self.filter(filter_)
-            n_pixels = filter_.metadata.number_of_pixels
-        else:
-            confusion_matrices = self._confusion_matrices
-            label_metadata = self._label_metadata
-            n_pixels = self.metadata.number_of_pixels
-
-        results = compute_metrics(
-            confusion_matrices=confusion_matrices,
-            label_metadata=label_metadata,
-            n_pixels=n_pixels,
-        )
-        return unpack_precision_recall_iou_into_metric_lists(
-            results=results,
-            label_metadata=label_metadata,
-            index_to_label=self.index_to_label,
-        )
-
-    def evaluate(
-        self, filter_: Filter | None = None
-    ) -> dict[MetricType, list[Metric]]:
-        """
-        Computes all available metrics.
-
-        Returns
-        -------
-        dict[MetricType, list[Metric]]
-            Lists of metrics organized by metric type.
-        """
-        return self.compute_precision_recall_iou(filter_=filter_)
-
-
-class DataLoader:
-    """
-    Segmentation DataLoader.
-    """
-
-    def __init__(self):
-        self._evaluator = Evaluator()
-        self.matrices = list()
-
-    def _add_datum(self, uid: str) -> int:
-        """
-        Helper function for adding a datum to the cache.
-
-        Parameters
-        ----------
-        uid : str
-            The datum uid.
-
-        Returns
-        -------
-        int
-            The datum index.
-        """
-        if uid in self._evaluator.datum_id_to_index:
-            raise ValueError(f"Datum with uid `{uid}` already exists.")
-        index = len(self._evaluator.datum_id_to_index)
-        self._evaluator.datum_id_to_index[uid] = index
-        self._evaluator.index_to_datum_id.append(uid)
-        return index
-
-    def _add_label(self, label: str) -> int:
-        """
-        Helper function for adding a label to the cache.
-
-        Parameters
-        ----------
-        label : str
-            A string label.
-
-        Returns
-        -------
-        int
-            The label's index.
-        """
-        if label not in self._evaluator.label_to_index:
-            label_id = len(self._evaluator.index_to_label)
-            self._evaluator.label_to_index[label] = label_id
-            self._evaluator.index_to_label.append(label)
-        return self._evaluator.label_to_index[label]
-
-    def add_data(
-        self,
-        segmentations: list[Segmentation],
-        show_progress: bool = False,
-    ):
-        """
-        Adds segmentations to the cache.
-
-        Parameters
-        ----------
-        segmentations : list[Segmentation]
-            A list of Segmentation objects.
-        show_progress : bool, default=False
-            Toggle for tqdm progress bar.
-        """
-
-        disable_tqdm = not show_progress
-        for segmentation in tqdm(segmentations, disable=disable_tqdm):
-            # update datum cache
-            self._add_datum(segmentation.uid)
-
-            groundtruth_labels = -1 * np.ones(
-                len(segmentation.groundtruths), dtype=np.int64
-            )
-            for idx, groundtruth in enumerate(segmentation.groundtruths):
-                label_idx = self._add_label(groundtruth.label)
-                groundtruth_labels[idx] = label_idx
-
-            prediction_labels = -1 * np.ones(
-                len(segmentation.predictions), dtype=np.int64
-            )
-            for idx, prediction in enumerate(segmentation.predictions):
-                label_idx = self._add_label(prediction.label)
-                prediction_labels[idx] = label_idx
-
-            if segmentation.groundtruths:
-                combined_groundtruths = np.stack(
-                    [
-                        groundtruth.mask.flatten()
-                        for groundtruth in segmentation.groundtruths
-                    ],
-                    axis=0,
-                )
-            else:
-                combined_groundtruths = np.zeros(
-                    (1, segmentation.shape[0] * segmentation.shape[1]),
-                    dtype=np.bool_,
-                )
-
-            if segmentation.predictions:
-                combined_predictions = np.stack(
-                    [
-                        prediction.mask.flatten()
-                        for prediction in segmentation.predictions
-                    ],
-                    axis=0,
-                )
-            else:
-                combined_predictions = np.zeros(
-                    (1, segmentation.shape[0] * segmentation.shape[1]),
-                    dtype=np.bool_,
-                )
-
-            self.matrices.append(
-                compute_intermediate_confusion_matrices(
-                    groundtruths=combined_groundtruths,
-                    predictions=combined_predictions,
-                    groundtruth_labels=groundtruth_labels,
-                    prediction_labels=prediction_labels,
-                    n_labels=len(self._evaluator.index_to_label),
-                )
-            )
-
-    def finalize(self) -> Evaluator:
-        """
-        Performs data finalization and some preprocessing steps.
-
-        Returns
-        -------
-        Evaluator
-            A ready-to-use evaluator object.
-        """
-
-        if len(self.matrices) == 0:
-            raise EmptyEvaluatorError()
-
-        n_labels = len(self._evaluator.index_to_label)
-        n_datums = len(self._evaluator.index_to_datum_id)
-        self._evaluator._confusion_matrices = np.zeros(
-            (n_datums, n_labels + 1, n_labels + 1), dtype=np.int64
-        )
-        for idx, matrix in enumerate(self.matrices):
-            h, w = matrix.shape
-            self._evaluator._confusion_matrices[idx, :h, :w] = matrix
-        self._evaluator._label_metadata = compute_label_metadata(
-            confusion_matrices=self._evaluator._confusion_matrices,
-            n_labels=n_labels,
-        )
-        self._evaluator._metadata = Metadata.create(
-            confusion_matrices=self._evaluator._confusion_matrices,
-        )
-        return self._evaluator

valor_lite-0.36.6.dist-info/RECORD

@@ -1,41 +0,0 @@
-valor_lite/LICENSE,sha256=M0L53VuwfEEqezhHb7NPeYcO_glw7-k4DMLZQ3eRN64,1068
-valor_lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valor_lite/exceptions.py,sha256=Q0PLMu0PnCPBx438iEPzpOQyMOcMOA3lOf5xQZP_yYU,385
-valor_lite/profiling.py,sha256=TLIROA1qccFw9NoEkMeQcrvvGGO75c4K5yTIWoCUix8,11746
-valor_lite/schemas.py,sha256=pB0MrPx5qFLbwBWDiOUUm-vmXdWvbJLFCBmKgbcbI5g,198
-valor_lite/classification/__init__.py,sha256=KXaVwyqAbeeeEq7bzNPyt4GTpbxhrABjV7lR58KR6Y4,440
-valor_lite/classification/annotation.py,sha256=0aUOvcwBAZgiNOJuyh-pXyNTG7vP7r8CUfnU3OmpUwQ,1113
-valor_lite/classification/computation.py,sha256=kB5n-RHzDsKG75Guvgg25xAOeLEQCq1TgjwHwfwbQ60,12010
-valor_lite/classification/manager.py,sha256=JZwA9sf-OG7p7uK5qIo-D711kSpBDDeTcXsPr1uuIBI,16884
-valor_lite/classification/metric.py,sha256=nSNWjoxQ1ou7gxTPOYxLNoUYf7avKQzJq3NHR9jzM48,11693
-valor_lite/classification/numpy_compatibility.py,sha256=roqtTetsm1_HxuaejrthQdydjsRIy-FpXpGb86cLh_E,365
-valor_lite/classification/utilities.py,sha256=jAcir7dW-o4I2gk_NEmlRr8j8Iniyyq9QT5j3PMxVHk,6435
-valor_lite/object_detection/__init__.py,sha256=eSrVAOpSykk1CfHXIKy1necplonUGxjyVKyDQ5UZoBQ,343
-valor_lite/object_detection/annotation.py,sha256=LVec-rIk408LuFxcOoIkPk0QZMWSSxbmsady4wapC1s,7007
-valor_lite/object_detection/computation.py,sha256=FPAOVOpvWlZemu9Lv4AAtrVQSMb_enbSxvupp22_2BY,24262
-valor_lite/object_detection/manager.py,sha256=y5QujUSXrI6FHmA6sg7JyQrvPgikxFW1q0Zudg841r0,30010
-valor_lite/object_detection/metric.py,sha256=sUYSZwXYfIyfmXG6_7Tje1_ZL_QwvecPq85jrGmwOWE,22739
-valor_lite/object_detection/utilities.py,sha256=jdY7ltjVcnjCB4S_tRyt3gthyGcPbVQKLK6ZZO6HW5s,10870
-valor_lite/semantic_segmentation/__init__.py,sha256=3YdItCThY_tW23IChCBm-R0zahnbZ06JDVjs-gQLVes,293
-valor_lite/semantic_segmentation/annotation.py,sha256=KrDqmSVzBsutxEdh0Kl8VqcEpTe4S68UdkVg-nyxcFg,4025
-valor_lite/semantic_segmentation/benchmark.py,sha256=uxd0SiDY3npsgU5pdeT4HvNP_au9GVRWzoqT6br9DtM,5961
-valor_lite/semantic_segmentation/computation.py,sha256=ZO0qAFmq8lN73UjCyiynSv18qQDtn35FNOmvuXY4rOw,7380
-valor_lite/semantic_segmentation/manager.py,sha256=h5w8Xl-O9gZxAzqT-ESofVE2th7d3cYahx4hHBic3pw,14256
-valor_lite/semantic_segmentation/metric.py,sha256=T9RfPJf4WgqGQTXYvSy08vJG5bjXXJnyYZeW0mlxMa8,7132
-valor_lite/semantic_segmentation/utilities.py,sha256=zgVmV8nyKWQK-T4Ov8cZFQzOmTKc5EL7errKFvc2H0g,2957
-valor_lite/text_generation/__init__.py,sha256=pGhpWCSZjLM0pPHCtPykAfos55B8ie3mi9EzbNxfj-U,356
-valor_lite/text_generation/annotation.py,sha256=O5aXiwCS4WjA-fqn4ly-O0MsTHoIOmqxqCaAp9IeI3M,1270
-valor_lite/text_generation/computation.py,sha256=hGDkPfzWY9SDTdozd-nArexJ3ZSNlCIWqHGoD8vO2Cc,18652
-valor_lite/text_generation/manager.py,sha256=C4QwvronGHXmYSkaRmUGy7TN0C0aeyDx9Hb-ClNYXK4,24810
-valor_lite/text_generation/metric.py,sha256=C9gbWejjOJ23JVLecuUhYW5rkx30NUCfRtgsM46uMds,10409
-valor_lite/text_generation/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-valor_lite/text_generation/llm/exceptions.py,sha256=w4eYSJIJQ_jWuCmquCB6ETr_st_LzbDRlhnlPeqwmfo,349
-valor_lite/text_generation/llm/generation.py,sha256=XKPjCxPUZHiWInQSO7wLOb0YtMFLu50s8rHZe1Yz0s0,28954
-valor_lite/text_generation/llm/instructions.py,sha256=fz2onBZZWcl5W8iy7zEWkPGU9N07ez6O7SxZA5M2xe4,34056
-valor_lite/text_generation/llm/integrations.py,sha256=-rTfdAjq1zH-4ixwYuMQEOQ80pIFzMTe0BYfroVx3Pg,6974
-valor_lite/text_generation/llm/utilities.py,sha256=bjqatGgtVTcl1PrMwiDKTYPGJXKrBrx7PDtzIblGSys,1178
-valor_lite/text_generation/llm/validators.py,sha256=Wzr5RlfF58_2wOU-uTw7C8skan_fYdhy4Gfn0jSJ8HM,2700
-valor_lite-0.36.6.dist-info/METADATA,sha256=tqYUOtCt6-fgLB6xLFE0zs80nLZAzuP20nUG6ny9PW0,5071
-valor_lite-0.36.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-valor_lite-0.36.6.dist-info/top_level.txt,sha256=9ujykxSwpl2Hu0_R95UQTR_l07k9UUTSdrpiqmq6zc4,11
-valor_lite-0.36.6.dist-info/RECORD,,