edgefirst-validator 4.2.1__py3-none-any.whl

edgefirst/validator/runners/onnx.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+import os
+import time
+import ctypes
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from edgefirst.validator.publishers.utils.logger import logger
+from edgefirst.validator.runners.core import Runner
+
+if TYPE_CHECKING:
+    from edgefirst.validator.evaluators import ModelParameters, TimerContext
+
+
+class ONNXRunner(Runner):
+    """
+    Loads and runs ONNX models for inference.
+
+    Parameters
+    ----------
+    model: Any
+        This is typically the path to the model (.onnx)
+        or the loaded ONNX model.
+    parameters: ModelParameters
+        These are the model parameters set from the command line.
+    metadata: dict
+        The model metadata which contains information for decoding
+        the model outputs.
+    timer: TimerContext
+        A timer object for handling validation timings for the model.
+
+    Raises
+    ------
+    ImportError
+        Missing onnxruntime library.
+    FileNotFoundError
+        Raised if the path to the model does not exist.
+    """
+
+    def __init__(
+        self,
+        model: Any,
+        parameters: ModelParameters,
+        metadata: dict,
+        timer: TimerContext
+    ):
+        super(ONNXRunner, self).__init__(model, parameters, timer=timer)
+
+        try:
+            import onnxruntime
+        except ImportError:
+            raise ImportError(
+                "onnxruntime or onnxruntime-gpu is needed to run ONNX models.")
+
+        if isinstance(model, str):
+            if not os.path.exists(model):
+                raise FileNotFoundError(
+                    "The model '{}' does not exist.".format(model))
+
+            providers = self.select_providers()
+            logger(f"Selected Providers: {providers}", code="INFO")
+            self.model = onnxruntime.InferenceSession(
+                model, providers=providers)
+
+        self.graph_name = self.model.get_modelmeta().graph_name
+        self.output_names = [x.name for x in self.model.get_outputs()]
+        outputs = self.model.get_outputs()
+        self.init_decoder(metadata=metadata, outputs=outputs)
+
+        if self.parameters.warmup > 0:
+            self.warmup()
+
+    @staticmethod
+    def check_tensorrt_runtime() -> list:
+        """
+        The following libraries are needed to run ONNX
+        with TensorrtExecutionProvider.
+
+        - "libnvinfer.so"
+        - "libnvinfer_plugin.so"
+        - "libnvonnxparser.so"
+
+        Returns
+        -------
+        list
+            A list of the libraries that are missing.
+        """
+        required_libs = ["libnvinfer.so",
+                         "libnvinfer_plugin.so", "libnvonnxparser.so"]
+        missing = []
+        for lib in required_libs:
+            try:
+                ctypes.CDLL(lib)
+            except OSError:
+                missing.append(lib)
+        return missing
+
+    def select_providers(self) -> list:
+        """
+        Specify the providers to load based on
+        the type of engine specified.
+
+        Returns
+        -------
+        list
+            A list of the selected providers to deploy.
+        """
+        import onnxruntime
+
+        selected_providers = ["CPUExecutionProvider"]
+        available_providers = onnxruntime.get_available_providers()
+        if self.parameters.engine.lower() == "npu":
+            preferred_providers = ["NnapiExecutionProvider",
+                                   "VsiNpuExecutionProvider",
+                                   "TensorrtExecutionProvider",
+                                   "CUDAExecutionProvider",
+                                   "CPUExecutionProvider"]
+            selected_providers = []
+            for i, provider in enumerate(preferred_providers):
+                if provider in available_providers:
+                    if provider == "TensorrtExecutionProvider":
+                        missing_libraries = self.check_tensorrt_runtime()
+                        if missing_libraries:
+                            logger(f"The libraries {missing_libraries} are " +
+                                   "needed for TensorrtExecutionProvider. " +
+                                   f"Falling back to {preferred_providers[i+1]}.",
+                                   code="WARNING")
+                            continue
+                    selected_providers.append(provider)
+                else:
+                    logger(f"{provider} is not present in the system. " +
+                           f"Falling back to {preferred_providers[i+1]}.",
+                           code="WARNING")
+            if selected_providers in [["TensorrtExecutionProvider",
+                                       "CUDAExecutionProvider",
+                                       "CPUExecutionProvider"],
+                                      ["CUDAExecutionProvider",
+                                       "CPUExecutionProvider"]]:
+                self.parameters.engine = "gpu"
+            elif selected_providers == ["CPUExecutionProvider"]:
+                self.parameters.engine = "cpu"
+        elif self.parameters.engine.lower() == "gpu":
+            preferred_providers = ["TensorrtExecutionProvider",
+                                   "CUDAExecutionProvider",
+                                   "CPUExecutionProvider"]
+            selected_providers = []
+            for i, provider in enumerate(preferred_providers):
+                if provider in available_providers:
+                    if provider == "TensorrtExecutionProvider":
+                        missing_libraries = self.check_tensorrt_runtime()
+                        if missing_libraries:
+                            logger(f"The libraries {missing_libraries} are " +
+                                   "needed for TensorrtExecutionProvider. " +
+                                   f"Falling back to {preferred_providers[i+1]}.",
+                                   code="WARNING")
+                            continue
+                    selected_providers.append(provider)
+                else:
+                    logger(f"{provider} is not present in the system. " +
+                           f"Falling back to {preferred_providers[i+1]}.",
+                           code="WARNING")
+            if selected_providers == ["CPUExecutionProvider"]:
+                self.parameters.engine = "cpu"
+                logger(
+                    "TensorrtExecutionProvider and CUDAExecutionProvider is " +
+                    "not present in the system. Falling back to CPUExecutionProvider.",
+                    code="WARNING")
+        return selected_providers
+
+    def run_single_instance(self, image: np.ndarray) -> Any:
+        """
+        Run ONNX inference on a single image and record the timings.
+
+        Parameters
+        ----------
+        image: np.ndarray
+            The input image after being preprocessed.
+            Typically this is an RGB image array.
+
+        Returns
+        -------
+        Any
+            This could either return detection outputs after NMS.
+                np.ndarray
+                    The prediction bounding boxes.. [[box1], [box2], ...].
+                np.ndarray
+                    The prediction labels.. [cl1, cl2, ...].
+                np.ndarray
+                    The prediction confidence scores.. [score, score, ...]
+                    normalized between 0 and 1.
+            This could also return segmentation masks.
+                np.ndarray
+        """
+        # Inference
+        with self.timer.time("inference"):
+            outputs = self.model.run(self.output_names,
+                                     {self.model.get_inputs()[0].name: image})
+
+        # Postprocessing
+        return self.postprocessing(outputs)
+
+    def get_input_type(self) -> np.dtype:
+        """
+        This returns the input type of the model for the
+        input with shape in the form
+        (batch size, channels, height, width) or
+        (batch size, height, width, channels).
+
+        Returns
+        -------
+        np.dtype
+            The input type of the model.
+        """
+        if "float16" in self.model.get_inputs()[0].type:
+            return np.float16
+        elif "float" in self.model.get_inputs()[0].type:
+            return np.float32
+        elif "uint8" in self.model.get_inputs()[0].type:
+            return np.uint8
+        elif "int8" in self.model.get_inputs()[0].type:
+            return np.int8
+        return self.model.get_inputs()[0].type
+
+    def get_input_shape(self) -> np.ndarray:
+        """
+        This fetches the model input shape.
+
+        Returns
+        -------
+        np.ndarray
+            The model input shape
+            (batch size, channels, height, width) or
+            (batch size, height, width, channels).
+        """
+        for input in self.model.get_inputs():
+            if len(input.shape) == 4:
+                if input.shape[1] == 3 or input.shape[-1] == 3:
+                    return input.shape
+        return self.model.get_inputs()[0].shape

edgefirst/validator/runners/processing/decode.py

@@ -0,0 +1,320 @@
+from typing import Tuple, Union, List
+import copy
+
+import numpy as np
+
+from edgefirst.validator.metrics.utils.math import sigmoid
+from edgefirst.validator.datasets.utils.transformations import xcycwh2xyxy
+
+
+def decode_mpk_boxes(
+        p: np.ndarray, anchors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Decodes ModelPack boxes into boxes and scores.
+
+    Parameters
+    ----------
+    p: np.ndarray
+        Raw model output with typical shapes such
+        as (1, 9, 15, 21) or (1, 17, 30, 21).
+    anchors: np.ndarray
+        Model anchors used for decoding the outputs
+        sometimes with shape (3, 3).
+
+    Returns
+    -------
+    boxes: np.ndarray
+        The decoded bounding boxes with shape (1, N, 1, 4).
+    scores: np.ndarray
+        The decoded scores with shape (1, N, nc) where nc is
+        the number of classes.
+    """
+    p = sigmoid(p)
+
+    na = anchors.shape[0]
+    nc = p.shape[-1] // na - 5
+    _, h, w, _ = p.shape
+
+    p = p.reshape((-1, h, w, na, nc + 5))
+    grid = np.meshgrid(np.arange(w), np.arange(h))
+    grid = np.expand_dims(np.stack(grid, axis=-1), axis=2)
+    grid = np.tile(np.expand_dims(grid, axis=0), [
+        1, 1, 1, na, 1])
+
+    # Decoding
+    xy = p[..., 0:2]
+    wh = p[..., 2:4]
+    obj = p[..., 4:5]
+    probs = p[..., 5:]
+
+    scores = obj * probs
+
+    xy = (xy * 2.0 + grid - 0.5) / (w, h)
+    wh = (wh * 2) ** 2 * anchors * 0.5
+    xyxy = np.concatenate([
+        xy - wh,
+        xy + wh
+    ], axis=-1)
+    xyxy = xyxy.reshape((1, -1, 1, 4))
+    scores = scores.reshape(1, -1, nc)
+
+    return xyxy, scores
+
+
+def decode_yolo_boxes(
+    p: np.ndarray,
+    with_masks: bool,
+    nc: int
+) -> Tuple[np.ndarray, np.ndarray, Union[np.ndarray, None]]:
+    """
+    Takes the output from Ultralytics models and decodes
+    boxes, scores, classes, and mask coefficients (segmentation models).
+
+    Parameters
+    ----------
+    p: np.ndarray
+        The model output tensor with shape (1, nc + 4, 8400) or
+        (1, nc + 4 + 32, 8400) for detection and segmentation respectively.
+    with_masks: bool
+        Slice the last 32 values from the output as the mask
+        proto coefficients.
+    nc: int
+        The number of labels.
+
+    Returns
+    -------
+    boxes: np.ndarray
+        The boxes tensor with shape (8400, 4).
+    scores: np.ndarray
+        The scores tensor with shape (8400,).
+    masks: Union[np.ndarray, None]
+        The masks tensor coefficients with shape (8400, 32).
+        Otherwise, if the model is detection, this will be None.
+    """
+    masks = None
+    if p.shape[0] == 1:
+        p = p[0]
+    # Only transpose if shapes are [116, 8400] or [85, 25200]
+    if p.shape[0] < p.shape[1]:
+        # Transposing shape (116, 8400) -> (8400, 116).
+        p = p.transpose((1, 0))
+    boxes = xcycwh2xyxy(boxes=p[:, 0:4])
+    if with_masks:
+        det_i = p.shape[1] - 32
+        scores = p[:, 4:det_i]
+        masks = p[:, det_i:]  # Additional 32 protos from segmentation models.
+    else:
+        # YOLOv5 models contains [x, y, x, y, obj_conf, cls_conf] outputs.
+        if p.shape[1] == nc + 5:
+            scores = p[:, 5:]
+            scores *= p[:, 4:5]  # conf = obj_conf * cls_conf # NOSONAR
+        # YOLOv8 and YOLOv11
+        else:
+            scores = p[:, 4:]
+    return boxes, scores, masks
+
+
+def decode_yolox_boxes(
+    p: np.ndarray,
+    shape: tuple,
+    p6: bool = False
+) -> Tuple[np.ndarray, np.ndarray]:
+    """
+    Decodes YOLOx outputs into boxes and scores.
+
+    Parameters
+    ----------
+    p: np.ndarray
+        The raw YOLOx model outputs with shape (1, 8400, 85).
+    shape: tuple
+        The model input shape (height, width).
+    p6: bool
+        If True, enables support for YOLOX-P6 with stride 64 detection head.
+
+    Returns
+    -------
+    boxes: np.ndarray
+        The decoded boxes with shape (8400, 4).
+    scores: np.ndarray
+        The decoded scores with shape (8400, nc).
+    """
+    h, w = shape
+
+    grids = []
+    expanded_strides = []
+    strides = [8, 16, 32] if not p6 else [8, 16, 32, 64]
+
+    hsizes = [h // stride for stride in strides]
+    wsizes = [w // stride for stride in strides]
+    for hsize, wsize, stride in zip(hsizes, wsizes, strides):
+        xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize))
+        grid = np.stack((xv, yv), 2).reshape(1, -1, 2)
+        grids.append(grid)
+        shape = grid.shape[:2]
+        expanded_strides.append(np.full((*shape, 1), stride))
+
+    grids = np.concatenate(grids, 1)
+    expanded_strides = np.concatenate(expanded_strides, 1)
+    p[..., :2] = (p[..., :2] + grids) * expanded_strides
+    p[..., 2:4] = np.exp(p[..., 2:4]) * expanded_strides
+    predictions = p[0]
+
+    boxes = predictions[:, :4]
+    scores = predictions[:, 4:5] * predictions[:, 5:]
+
+    return boxes, scores
+
+
+def decode_yolo_masks(masks: np.ndarray, protos: np.ndarray) -> np.ndarray:
+    """
+    Takes the output from Ultralytics segmentation models and
+    decodes instance segmentation masks from the model.
+
+    Parameters
+    ----------
+    masks: np.ndarray
+        The mask coefficients with shape (n, 32).
+    protos: np.ndarray
+        The raw output mask tensor with shape (1, h, w, 32).
+
+    Returns
+    -------
+    np.ndarray
+        The instance mask per object with shape (n, h, w).
+    """
+    # In case of shape (1, 32, h, w).
+    if protos.shape[1] == 32:
+        c, h, w = protos[0].shape
+    else:
+        h, w, c = protos[0].shape
+        protos = np.transpose(protos, (0, 3, 1, 2))
+    masks = np.matmul(masks, protos.reshape(c, -1)).reshape(-1, h, w)
+    return masks
+
+
+def decode_mpk_masks(masks: np.ndarray) -> np.ndarray:
+    """
+    Decodes ModelPack masks into semantic segmentation.
+
+    Parameters
+    ----------
+    masks: np.ndarray
+        The raw segmentation masks from the model
+        with shape (1, h, w, nc).
+
+    Returns
+    -------
+    np.ndarray
+        The decoded semantic segmentation mask
+        with shape (1, h, w).
+    """
+    if len(masks.shape) == 3:
+        return np.array(masks, dtype=np.uint8)
+    return np.argmax(masks, axis=-1).astype(np.uint8)
+
+
+def crop_masks(masks: np.ndarray, boxes: np.ndarray) -> np.ndarray:
+    """
+    Crops each instance mask to the bounding box.
+
+    Parameters
+    ----------
+    masks: np.ndarray
+        The instance mask per object with shape (n, h, w).
+    boxes: np.ndarray
+        Normalized box coordinates in [xmin, ymin, xmax, ymax]
+        format with shape (n, 4).
+
+    Returns
+    -------
+    np.ndarray
+        The instance mask per object cropped to the box dimensions
+        with shape (n, h, w).
+    """
+    _, h, w = masks.shape
+    x1, y1, x2, y2 = np.split(
+        boxes[:, :, np.newaxis], 4, axis=1)  # shape (n, 1, 1)
+    r = np.arange(w, dtype=boxes.dtype)[None, None, :]  # rows shape(1,1,w)
+    c = np.arange(h, dtype=boxes.dtype)[None, :, None]  # cols shape(1,h,1)
+    return masks * ((r >= x1 * w) * (r < x2 * w)
+                    * (c >= y1 * h) * (c < y2 * h))
+
+
+def dequantize(x: np.ndarray, scale: float, zero_point: float) -> np.ndarray:
+    """
+    Dequantization of the model output tensor based on the scale
+    and zero point values.
+
+    Parameters
+    ----------
+    x: np.ndarray
+        Quantized model output tensor typically with uint8 or int8 dtypes.
+    scale: float
+        Quantization scale factor.
+    zero_point: float
+        Quantization shift factor for signed tensors.
+
+    Returns
+    -------
+    np.ndarray
+        Dequantized tensors typically float32 dtypes.
+    """
+    if scale > 0:
+        x = (x.astype(np.float32) - zero_point) * scale  # re-scale
+    return x
+
+
+def dequantize_kinara(
+    output_dict: dict,
+    method: str = "hal"
+) -> List[np.ndarray]:
+    """
+    Dequantize a list of quantized outputs from Kinara models.
+
+    Parameters
+    ----------
+    output_list : dict
+        List of quantized model outputs.
+    method: str
+        The method to use for performing the dequantization.
+        By default "hal" is used. Otherwise, NumPy operations are used.
+
+    Returns
+    -------
+    list of np.ndarray
+        List of dequantized outputs as float arrays.
+    """
+    num_outputs = len(output_dict)
+    dequantized_outputs = []
+    for i in range(num_outputs):
+        output: np.ndarray = output_dict[i].numpy_data
+        out_param = output_dict[i].params
+
+        if (not out_param.postprocess_param.is_struct_format
+                and not out_param.postprocess_param.is_float):
+
+            bpp = out_param.bpp
+            dtype = np.int8
+            if bpp == 1:
+                dtype = np.int8 if out_param.postprocess_param.is_signed else np.uint8
+            elif bpp == 2:
+                dtype = np.int16 if out_param.postprocess_param.is_signed else np.uint16
+            elif bpp == 4:
+                dtype = np.int32 if out_param.postprocess_param.is_signed else np.uint32
+
+            qn, offset = (out_param.postprocess_param.qn,
+                          out_param.postprocess_param.offset)
+            buf = output.view(dtype=dtype)
+
+            if method == "hal":
+                dst_buffer = buf
+            else:
+                dst_buffer = (
+                    (buf.astype(int) -
+                     offset) *
+                    qn).astype(
+                    np.float32)
+            dequantized_outputs.append(dst_buffer)
+        else:
+            dequantized_outputs.append(output)
+    return dequantized_outputs