edgefirst-validator 4.2.1__py3-none-any.whl

edgefirst/validator/evaluators/utils/__init__.py

@@ -0,0 +1,3 @@
+from edgefirst.validator.evaluators.utils.match import Matcher
+from edgefirst.validator.evaluators.utils.timer import TimerContext
+from edgefirst.validator.evaluators.utils.classify import DetectionClassifier

edgefirst/validator/evaluators/utils/classify.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Tuple
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from edgefirst.validator.evaluators import ValidationParameters
+    from edgefirst.validator.datasets import DetectionInstance
+    from edgefirst.validator.metrics import DetectionStats
+    from edgefirst.validator.evaluators import Matcher
+    from edgefirst.validator.metrics import Plots
+
+
+class DetectionClassifier:
+    """
+    Classifies model predictions into true positives, false positives, or
+    false negatives depending on the results of the matching algorithm
+    and the labels. Furthermore, the confusion matrix data is also being
+    collected in this stage.
+
+    Parameters
+    ----------
+    parameters: ValidationParameters
+        This contains the validation parameters set from the command line.
+    detection_stats: DetectionStats
+        This stores the number of true positives, false positives, and
+        false negatives per label found throughout validation.
+    matcher: Matcher
+        The matcher object that matches the predictions to the
+        ground truth and contains matching results.
+    plots: PlotSummary
+        This is a container for the data to draw the plots.
+    """
+
+    def __init__(
+        self,
+        parameters: ValidationParameters,
+        detection_stats: DetectionStats,
+        matcher: Matcher,
+        plots: Plots
+    ):
+        self.parameters = parameters
+        self.detection_stats = detection_stats
+        self.matcher = matcher
+        self.plots = plots
+
+    def classify(
+        self,
+        gt_instance: DetectionInstance,
+        dt_instance: DetectionInstance
+    ):
+        """
+        Classifies the matched, missed, and extra detections
+        into true positives, localization and classification false positives,
+        and false negatives.
+
+        Parameters
+        ----------
+        gt_instance: DetectionInstance
+            The ground truth instance that contains the
+            ground truth boxes and labels.
+        dt_instance: DetectionInstance
+            The detection instance that contains the model
+            prediction boxes, labels, and scores for the image.
+        """
+        self.classify_matches(gt_instance=gt_instance, dt_instance=dt_instance)
+        self.classify_unmatched_dt(dt_instance)
+        self.classify_unmatched_gt(gt_instance)
+        self.setup_yolo_map(gt_instance=gt_instance, dt_instance=dt_instance)
+
+    def classify_matches(
+        self,
+        gt_instance: DetectionInstance,
+        dt_instance: DetectionInstance
+    ):
+        """
+        Classifies the matching ground truth to detection boxes
+        into true positives or classification false positives.
+
+        Parameters
+        ----------
+        gt_instance: DetectionInstance
+            The ground truth instance that contains the
+            ground truth boxes and labels.
+        dt_instance: DetectionInstance
+            The detection instance that contains the model
+            prediction boxes, labels, and scores for the image.
+        """
+        for match in self.matcher.index_matches:
+            dt_label = dt_instance.labels[match[0]]
+            gt_label = gt_instance.labels[match[1]]
+            score = dt_instance.scores[match[0]]
+            iou = self.matcher.iou_list[match[0]]
+
+            if dt_label != gt_label:
+                label_data = self.detection_stats.get_label_data(dt_label)
+                if label_data:
+                    label_data.add_cfp(iou, score)
+
+            label_data = self.detection_stats.get_label_data(gt_label)
+            if label_data:
+                label_data.add_ground_truths()
+                if dt_label == gt_label:
+                    label_data.add_tp(iou, score)
+
+            # Format confusion matrix
+            if isinstance(gt_label, str):
+                gt_label = self.plots.confusion_labels.index(gt_label)
+            else:
+                gt_label = self.plots.confusion_labels.index(
+                    self.plots.labels[int(gt_label)])
+
+            if isinstance(dt_label, str):
+                dt_label = self.plots.confusion_labels.index(dt_label)
+            else:
+                dt_label = self.plots.confusion_labels.index(
+                    self.plots.labels[int(dt_label)])
+
+            if iou >= self.parameters.iou_threshold:
+                self.plots.confusion_matrix[dt_label, gt_label] += 1
+            else:
+                self.plots.confusion_matrix[dt_label, 0] += 1  # False positive
+                self.plots.confusion_matrix[0, gt_label] += 1  # False negative
+
+    def classify_unmatched_dt(self, dt_instance: DetectionInstance):
+        """
+        Classifies the extra predictions into localization false positives.
+
+        Parameters
+        ----------
+        dt_instance: DetectionInstance
+            The detection instance that contains the model
+            prediction boxes, labels, and scores for the image.
+        """
+        for extra in self.matcher.index_unmatched_dt:
+            dt_label = dt_instance.labels[extra]
+            score = dt_instance.scores[extra]
+
+            label_data = self.detection_stats.get_label_data(dt_label)
+            if label_data:
+                label_data.add_lfp(score)
+
+            # Format confusion matrix
+            if isinstance(dt_label, str):
+                dt_label = self.plots.confusion_labels.index(dt_label)
+            else:
+                dt_label = self.plots.confusion_labels.index(
+                    self.plots.labels[int(dt_label)])
+            self.plots.confusion_matrix[dt_label, 0] += 1  # False positive
+
+    def classify_unmatched_gt(self, gt_instance: DetectionInstance):
+        """
+        Classifies the missed predictions into false negatives.
+
+        Parameters
+        ----------
+        gt_instance: DetectionInstance
+            The ground truth instance that contains the
+            ground truth boxes and labels.
+        """
+        for miss in self.matcher.index_unmatched_gt:
+            gt_label = gt_instance.labels[miss]
+
+            label_data = self.detection_stats.get_label_data(gt_label)
+            if label_data:
+                label_data.add_ground_truths()
+
+            # Format confusion matrix
+            if isinstance(gt_label, str):
+                gt_label = self.plots.confusion_labels.index(gt_label)
+            else:
+                gt_label = self.plots.confusion_labels.index(
+                    self.plots.labels[int(gt_label)])
+            self.plots.confusion_matrix[0, gt_label] += 1  # False negative
+
+    def setup_yolo_map(
+        self,
+        gt_instance: DetectionInstance,
+        dt_instance: DetectionInstance
+    ):
+        """
+        Formulates the variables needed to utilize the functionality of
+        calculating the average percision per class in YOLOv5.
+        https://github.com/ultralytics/yolov5/blob/master/utils/metrics.py#L29.
+
+        The following parameters are followed:
+
+        tp: (nx10) np.ndarray
+            This contains the True and False for each detection (rows) for
+            each IoU at 0.50-0.95 (columns).
+        conf: (nx1) np.ndarray
+            The confidence scores of each detections.
+        pred_cls: (nx1) np.ndarray
+            The prediction classes.
+        target_cls: (nx1) np.ndarray
+            The ground truth classes.
+
+        Parameters
+        ----------
+        gt_instance: DetectionInstance
+            The ground truth instance that contains the
+            ground truth boxes and labels.
+        dt_instance: DetectionInstance
+            The detection instance that contains the model
+            prediction boxes, labels, and scores for the image.
+        """
+        for match in self.matcher.index_matches:
+            dt_label = dt_instance.labels[match[0]]
+            gt_label = gt_instance.labels[match[1]]
+            score = dt_instance.scores[match[0]]
+            iou = self.matcher.iou_list[match[0]]
+
+            if dt_label != gt_label:
+                tp = [False for _ in self.detection_stats.ious]
+            else:
+                tp = [iou >= x for x in self.detection_stats.ious]
+
+            self.detection_stats.tp.append(tp)
+            self.detection_stats.conf.append(score)
+            self.detection_stats.pred_cls.append(dt_label)
+            self.detection_stats.target_cls.append(gt_label)
+
+        for extra in self.matcher.index_unmatched_dt:
+            dt_label = dt_instance.labels[extra]
+            score = dt_instance.scores[extra]
+
+            tp = [False for _ in self.detection_stats.ious]
+            self.detection_stats.tp.append(tp)
+            self.detection_stats.conf.append(score)
+            self.detection_stats.pred_cls.append(dt_label)
+
+        for miss in self.matcher.index_unmatched_gt:
+            gt_label = gt_instance.labels[miss]
+            self.detection_stats.target_cls.append(gt_label)
+
+
+def classify_mask(
+    gt_class_mask: np.ndarray,
+    dt_class_mask: np.ndarray,
+    exclude_background: np.ndarray = True
+) -> Tuple[int, int, int]:
+    """
+    Classifies if the pixels are either true predictions or false predictions.
+    Note the masks provided can also be multiclass, however this function
+    is used primarily to find the true predictions and false predictions
+    per class.
+
+    Parameters
+    ----------
+    gt_class_mask: (height, width) np.ndarray
+        2D binary array representing pixels forming the image ground truth.
+        1 represents the class being classified and 0 are the rest of
+        the classes.
+    dt_class_mask: (height, width) np.ndarray
+        2D binary array representing pixels forming the image prediction.
+        1 represents the class being classified and 0 are the rest of
+        the classes.
+    exclude_background: bool
+        Specify to avoid background to background
+        predictions and ground truths as true predictions.
+
+    Returns
+    -------
+    true_predictions: int
+        The number of true predictions pixels in the image.
+    false_predictions: int
+        The number of false predictions pixels in the image.
+    union: int
+        The union between ground truths and model predictions occurs
+        when both arrays are non-zero. The union is the sum of
+        true predictions and false predictions.
+    """
+    gt_mask_flat = gt_class_mask.flatten()
+    dt_mask_flat = dt_class_mask.flatten()
+
+    if exclude_background:
+        # Do not consider 0 against 0 as true predictions. 0 means another class
+        # not just background. True predictions are 1 against 1 which means this
+        # current class.
+        true_predictions = np.sum(
+            (gt_mask_flat == dt_mask_flat) & (gt_mask_flat > 0) & (dt_mask_flat > 0))
+
+        # The union between ground truths and predictions where both are
+        # non-zero.
+        union = np.sum((gt_mask_flat != 0) | (dt_mask_flat != 0))
+    else:
+        true_predictions = np.sum(gt_mask_flat == dt_mask_flat)
+        union = len(gt_mask_flat)
+
+    false_predictions = np.sum(gt_mask_flat != dt_mask_flat)
+    return true_predictions, false_predictions, union

edgefirst/validator/evaluators/utils/match.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, Union
+
+import numpy as np
+
+from edgefirst.validator.metrics.utils.math import iou_2d, localize_distance
+
+if TYPE_CHECKING:
+    from edgefirst.validator.evaluators import ValidationParameters
+
+
+class Matcher:
+    """
+    The Matching Algorithm used in EdgeFirst Validation. This algorithm
+    will run matching recursively to find the best matches based on IoU
+    with a preference to a matching ground truth and detection labels.
+    The Matching and Classification rules is documented in::
+    https://au-zone.atlassian.net/wiki/spaces/DV/pages/2325938299/DeepView-Validator+Matching+and+Classification+Rules
+
+    Parameters
+    ----------
+    parameters: ValidationParameters
+        This contains the validation parameters set from the command line.
+    """
+
+    def __init__(self, parameters: ValidationParameters):
+        self.parameters = parameters
+        self.gt_boxes = list()
+        self.gt_labels = list()
+        self.dt_boxes = list()
+        self.dt_labels = list()
+
+        # This contains the IoUs of each detection to ground truth match.
+        self.iou_list = list()
+        # An IoU grid where rows are the ground truths
+        # and the predictions are the columns.
+        self.iou_grid = list()
+        # The matches containing ground truth and detection indices:
+        # [[gti, dti], [gti, dti], ..].
+        self.index_matches = list()
+        # The prediction indices that were not matched.
+        self.index_unmatched_dt = list()
+        # The ground truth indices that were not matched.
+        self.index_unmatched_gt = list()
+
+    def set_boxes(
+        self,
+        gt_boxes: list,
+        gt_labels: list,
+        dt_boxes: list,
+        dt_labels: list
+    ):
+        """
+        Sets the bounding boxes and labels for the ground truth and model
+        predictions to match the bounding boxes described in the index_matches.
+        Calling this method will also reset the previous matching results.
+
+        Parameters
+        ----------
+        gt_boxes: list
+            The ground truth bounding boxes in the format [xmin, ymin, xmax, ymax].
+        gt_labels: list
+            The ground truth labels for each bounding box. This can either
+            contain strings or integers.
+        dt_boxes: list
+            The prediction bounding boxes in the format [xmin, ymin, xmax, ymax].
+        dt_labels: list
+            The prediction labels for each bounding box. This can either
+            contain strings or integers.
+        """
+        # Setting boxes requires reset for a new matching process.
+        self.reset()
+        self.gt_boxes = gt_boxes
+        self.gt_labels = gt_labels
+        self.dt_boxes = dt_boxes
+        self.dt_labels = dt_labels
+
+        self.iou_list = np.zeros(len(self.dt_boxes))
+        self.iou_grid = np.zeros((len(self.gt_boxes), len(self.dt_boxes)))
+
+        # The prediction indices that were not matched.
+        self.index_unmatched_dt = list(range(0, len(self.dt_boxes)))
+        # The ground truth indices that were not matched.
+        self.index_unmatched_gt = list(range(0, len(self.gt_boxes)))
+
+    def match(
+        self,
+        gt_boxes: list,
+        gt_labels: list,
+        dt_boxes: list,
+        dt_labels: list
+    ):
+        """
+        The matching algorithm which matches the predictions to ground truth
+        based on matching labels first and then by highest IoU or lowest
+        centerpoint distance between boxes.
+
+        This algorithm also incorporates recursive calls to
+        perform rematching of ground truth that were unmatched due to
+        duplicative matches, but the rematching is based on the next best IoU.
+
+        Parameters
+        ----------
+        gt_boxes: list
+            The ground truth bounding boxes in the format [xmin, ymin, xmax, ymax].
+        gt_labels: list
+            The ground truth labels for each bounding box. This can either
+            contain strings or integers.
+        dt_boxes: list
+            The prediction bounding boxes in the format [xmin, ymin, xmax, ymax].
+        dt_labels: list
+            The prediction labels for each bounding box. This can either
+            contain strings or integers.
+        """
+        self.set_boxes(
+            gt_boxes=gt_boxes,
+            gt_labels=gt_labels,
+            dt_boxes=dt_boxes,
+            dt_labels=dt_labels
+        )
+
+        if 0 in [len(self.gt_boxes), len(self.dt_boxes)]:
+            return
+
+        for gti, gt in enumerate(self.gt_boxes):
+            # A list of prediction indices with
+            # matching labels as the ground truth.
+            dti_reflective, iou_reflective = list(), list()
+            gt_label = self.gt_labels[gti]
+
+            for dti, dt in enumerate(self.dt_boxes):
+                self.iou_grid[gti][dti] = self.get_metric(gt, dt)
+
+                dt_label = self.dt_labels[dti]
+                if dt_label == gt_label:
+                    dti_reflective.append(dti)
+                    iou_reflective.append(self.iou_grid[gti][dti])
+
+            # A potential match is the detection that produced the highest IoU.
+            dti = np.argmax(self.iou_grid[gti])
+            iou = max(self.iou_grid[gti])
+            # If there is no intersection, it cannot be a match.
+            if iou < 0:
+                continue
+            # Only match if the IoU between matching ground truth and detection
+            # labels > 0.
+            if len(dti_reflective) and max(
+                    iou_reflective) >= self.parameters.iou_threshold:
+                # The IoU of the detections with the same labels
+                # as the ground truth. A potential match is the
+                # detection with the same label as the ground truth.
+                dti = dti_reflective[np.argmax(iou_reflective)]
+                iou = max(iou_reflective)
+            self.compare_matches(dti, gti, iou)
+
+        # Find the unmatched predictions
+        for match in self.index_matches:
+            self.index_unmatched_dt.remove(match[0])
+            self.index_unmatched_gt.remove(match[1])
+
+    def compare_matches(self, dti: int, gti: int, iou: float):
+        """
+        Checks if duplicate matches exists. A duplicate match is when the
+        same detection is being matched to more than one ground truth.
+        The IoUs are compared and the better IoU is the true match and the
+        ground truth of the other match is then rematch to the next best IoU,
+        but it performs a recursive call to check if the next best IoU
+        also generates a duplicate match.
+
+        Parameters
+        ----------
+        dti: int
+            The detection index being matched to the current ground truth.
+        gti: int
+            The current ground truth matched to the detection.
+        iou: float
+            The current best IoU that was computed for the current ground
+            truth against all detections.
+        """
+        twice_matched = [(d, g) for d, g in self.index_matches if d == dti]
+        assert len(twice_matched) < 2, "More than two duplicate matches occurred."
+
+        if len(twice_matched) == 1:
+            # Compare the IoUs between duplicate matches.
+            dti, pre_gti = twice_matched[0]
+            if iou > self.iou_list[dti]:
+                self.index_matches.remove((dti, pre_gti))
+                self.iou_list[dti] = iou
+                self.index_matches.append((dti, gti))
+
+                # Rematch pre_gti
+                self.iou_grid[pre_gti][dti] = 0.
+                dti = np.argmax(self.iou_grid[pre_gti])
+                iou = max(self.iou_grid[pre_gti])
+                if iou > 0:
+                    self.compare_matches(dti, pre_gti, iou)
+            else:
+                # Rematch gti
+                self.iou_grid[gti][dti] = 0.
+                dti = np.argmax(self.iou_grid[gti])
+                iou = max(self.iou_grid[gti])
+                if iou > 0:
+                    self.compare_matches(dti, gti, iou)
+        else:
+            if iou > 0:
+                self.iou_list[dti] = iou
+                self.index_matches.append((dti, gti))
+
+    def get_metric(
+        self,
+        gt: Union[list, np.ndarray],
+        dt: Union[list, np.ndarray],
+    ) -> float:
+        """
+        Computes either the 3D or 2D IoU or centerpoint distances
+        and stores the values in the IoU grid.
+
+        When the iou_first flag is False, IoU is
+        considered 0 if the classes don't match.
+
+        Parameters
+        ----------
+        gt: Union[list, np.ndarray]
+            This either contains ground truth bounding boxes
+            if 2D validation or 3D box corners if 3D validation.
+        dt: Union[list, np.ndarray]
+            This either contains prediction bounding boxes
+            if 2D validation or 3D box corners if 3D validation.
+
+        Returns
+        -------
+        float
+            The IoU or centerpoint distance between two boxes.
+
+        Raises
+        ------
+        TypeError
+            Raised if an invalid metric type is specified.
+        """
+        if self.parameters.metric == "iou":
+            return iou_2d(dt.astype(float), gt.astype(float))
+        elif self.parameters.metric == "centerpoint":
+            return 1 - localize_distance(
+                dt.astype(float),
+                gt.astype(float),
+                leniency_factor=self.parameters.matching_leniency
+            )
+        else:
+            raise TypeError(
+                "Unknown matching matching metric specified: {}".format(
+                    self.parameters.metric
+                ))
+
+    def reset(self):
+        """
+        Resets the containers to allow for a new matching process.
+        """
+        self.iou_list = list()
+        self.iou_grid = list()
+        self.index_matches = list()
+        self.index_unmatched_dt = list()
+        self.index_unmatched_gt = list()

edgefirst/validator/evaluators/utils/timer.py

@@ -0,0 +1,132 @@
+import time
+from contextlib import contextmanager
+
+import numpy as np
+
+
+class TimerContext:
+    """
+    This class provides methods for timing measurements of the
+    validation process and acts as a container of the measured timings.
+    The stages of timings are as follows:
+
+    * input => Image read and preprocessing
+    * inference => Inference Timings
+    * output => Output decoding and postprocessing
+    """
+
+    def __init__(self):
+        self.stages = ["input", "inference", "output"]
+        self.__timings = {stage: [] for stage in self.stages}
+        self.__start_time = None
+        self.to_ms = 1e3
+
+    @contextmanager
+    def time(self, stage: str):
+        """
+        Context manager to time a section
+        and store duration in list.
+
+        Parameters
+        ----------
+        stage: str
+            The key to store the timing measurement
+            for a specific stage such as "input", "inference",
+            or "output".
+        """
+        self.__start_time = time.perf_counter()
+        yield
+        elapsed = time.perf_counter() - self.__start_time
+        self.__timings[stage].append(elapsed * self.to_ms)
+        self.__start_time = None
+
+    def add_time(self, stage: str, ds_ms: float):
+        """
+        Add time to the last timining measurement
+        of the current stage in the timings.
+
+        Parameters
+        ----------
+        stage: str
+            The key to store the timing measurement
+            for a specific stage such as "input", "inference",
+            or "output".
+        ds_ms: float
+            The time duration in milli seconds to add.
+        """
+        if len(self.__timings[stage]):
+            self.__timings[stage][-1] += ds_ms
+
+    def get_average_time(self, stage: str) -> float:
+        """
+        Returns the average time for the specific stage.
+
+        Parameters
+        ----------
+        stage: str
+            The key to fetch the timing measurements
+            for a specific stage such as "input", "inference",
+            or "output".
+
+        Returns
+        -------
+        float
+            The average timings for a given stage.
+        """
+        times = self.__timings.get(stage, [])
+        return np.mean(times) if len(times) else 0.0
+
+    def get_max_time(self, stage: str) -> float:
+        """
+        Returns the maximum time for the specific stage.
+
+        Parameters
+        ----------
+        stage: str
+            The key to fetch the timing measurements
+            for a specific stage such as "input", "inference",
+            or "output".
+
+        Returns
+        -------
+        float
+            The maximum timing for a given stage.
+        """
+        times = self.__timings.get(stage, [])
+        return np.max(times) if len(times) else 0.0
+
+    def get_min_time(self, stage: str) -> float:
+        """
+        Returns the minimum time for the specific stage.
+
+        Parameters
+        ----------
+        stage: str
+            The key to fetch the timing measurements
+            for a specific stage such as "input", "inference",
+            or "output".
+
+        Returns
+        -------
+        float
+            The minimum timing for a given stage.
+        """
+        times = self.__timings.get(stage, [])
+        return np.min(times) if len(times) else 0.0
+
+    def to_dict(self) -> dict:
+        """
+        Grabs the timing summary such as the average, max, min
+        for each stages and stores it as a dictionary.
+        """
+        timings = {}
+        for stage in ["input", "inference", "output"]:
+            timings[f"min_{stage}_time"] = self.get_min_time(stage)
+            timings[f"max_{stage}_time"] = self.get_max_time(stage)
+            timings[f"avg_{stage}_time"] = self.get_average_time(stage)
+        return timings
+
+    def reset(self):
+        """Clears all stored timings."""
+        self.__timings = {stage: [] for stage in self.stages}
+        self.__start_time = None

edgefirst/validator/metrics/__init__.py

@@ -0,0 +1,9 @@
+from edgefirst.validator.metrics.data.label import (DetectionLabelData,
+                                                    SegmentationLabelData)
+from edgefirst.validator.metrics.data.stats import (YOLOStats,
+                                                    DetectionStats,
+                                                    SegmentationStats)
+from edgefirst.validator.metrics.data.plots import Plots, MultitaskPlots
+from edgefirst.validator.metrics.data.metrics import Metrics, MultitaskMetrics
+from edgefirst.validator.metrics.detection import DetectionMetrics
+from edgefirst.validator.metrics.segmentation import SegmentationMetrics