edgefirst-validator 4.2.1__py3-none-any.whl

edgefirst/validator/runners/kinara.py

@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+import os
+import time
+import threading
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from edgefirst.validator.runners.processing.decode import dequantize_kinara
+from edgefirst.validator.runners.core import Runner
+
+if TYPE_CHECKING:
+    from edgefirst.validator.evaluators import ModelParameters, TimerContext
+
+
+class KinaraRunner(Runner):
+    """
+    Loads and runs Kinara models for inference.
+
+    Parameters
+    ----------
+    model: Any
+        This is typically the path to the model (.dvm)
+        or the loaded Kinara model.
+    parameters: ModelParameters
+        These are the model parameters set from the command line.
+    metadata: dict
+        The model metadata which contains information for decoding
+        the model outputs.
+    timer: TimerContext
+        A timer object for handling validation timings for the model.
+    socket_path : str, optional
+                Path to the UNIX socket for DVSession.
+
+    Raises
+    ------
+    ImportError
+        Missing Kinara library.
+    FileNotFoundError
+        Raised if the path to the model does not exist.
+    """
+
+    def __init__(
+        self,
+        model: Any,
+        parameters: ModelParameters,
+        metadata: dict,
+        timer: TimerContext,
+        socket_path: str = "/var/run/ara2.sock"
+    ):
+        super(KinaraRunner, self).__init__(model, parameters, timer)
+
+        self.socket_path = socket_path
+        self.conn = None
+        self.endpoints = None
+        self.loaded_model = None
+        self._lock = threading.Lock()
+        self.start()
+
+        if isinstance(model, str):
+            if not os.path.exists(model):
+                raise FileNotFoundError(
+                    "The model '{}' does not exist.".format(model))
+            self.load_model(model)
+
+        self.input_param = self.model.input_param[0]
+        self.preprocess_param = self.input_param.preprocess_param
+        self.parameters.common.input_quantization = (
+            self.preprocess_param.qn, self.preprocess_param.offset)
+
+        outputs = []
+        for output_param in self.model.output_param:
+            bpp = output_param.bpp
+            dtype = np.int8
+            if bpp == 1:
+                dtype = (np.int8 if output_param.postprocess_param.is_signed
+                         else np.uint8)
+            elif bpp == 2:
+                dtype = (np.int16 if output_param.postprocess_param.is_signed
+                         else np.uint16)
+            elif bpp == 4:
+                dtype = (np.int32 if output_param.postprocess_param.is_signed
+                         else np.uint32)
+
+            outputs.append(
+                {
+                    "shape": np.array([1, output_param.nch,
+                                       output_param.height]),
+                    "quantization": [output_param.postprocess_param.qn,
+                                     output_param.postprocess_param.offset],
+                    "channels_first": False,
+                    "dtype": dtype
+                }
+            )
+
+        self.init_decoder(metadata=metadata, outputs=outputs)
+        if self.parameters.warmup > 0:
+            self.warmup()
+
+    def start(self):
+        """
+        Start DVSession and fetch available endpoints.
+
+        Raises
+        ------
+        RuntimeError
+            If session creation or endpoint retrieval fails.
+        """
+        from edgefirst.validator.runners.processing.dvapi import DVSession
+
+        ret, self.conn = DVSession.create_via_unix_socket(self.socket_path)
+        if ret != 0:
+            raise RuntimeError("Failed to create DVSession")
+        ret, self.endpoints = self.conn.get_endpoint_list()
+        if ret != 0:
+            raise RuntimeError("Failed to get endpoints")
+
+    def load_model(self, model_path: str):
+        """
+        Load model from file into DVM endpoint.
+
+        Parameters
+        ----------
+        model_path : str
+            Path to the model file to load.
+
+        Raises
+        ------
+        RuntimeError
+            If model loading fails.
+        """
+        with self._lock:
+            ret, self.model = self.conn.load_model_from_file(
+                endpoint=self.endpoints[0], model_path=model_path)
+            if ret != 0:
+                raise RuntimeError("Failed to load model")
+
+    def infer(self, input_tensor, timeout: int = 50000):
+        """
+        Run synchronous inference on input tensor.
+
+        Parameters
+        ----------
+        image : DVTensor
+            Input tensor for inference.
+        timeout : int, optional
+            Timeout in milliseconds for inference.
+
+        Returns
+        -------
+        List[DVTensor]
+                Inference response outputs from the model.
+
+        Raises
+        ------
+        RuntimeError
+                If inference execution fails.
+        """
+        # thread-safe single-call wrapper
+        with self._lock:
+            ret, response = self.model.infer_sync(
+                [input_tensor], timeout=timeout, endpoint=self.endpoints[0])
+        if ret != 0:
+            raise RuntimeError("Inference failed")
+        return response
+
+    def run_single_instance(self, image: np.ndarray) -> Any:
+        """
+        Run Kinara inference on a single image and record the timings.
+
+        Parameters
+        ----------
+        image: np.ndarray
+            The input image after being preprocessed.
+            Typically this is an RGB image array.
+
+        Returns
+        -------
+        Any
+            This could either return detection outputs after NMS.
+                np.ndarray
+                    The prediction bounding boxes.. [[box1], [box2], ...].
+                np.ndarray
+                    The prediction labels.. [cl1, cl2, ...].
+                np.ndarray
+                    The prediction confidence scores.. [score, score, ...]
+                    normalized between 0 and 1.
+            This could also return segmentation masks.
+                np.ndarray
+        """
+        from edgefirst.validator.runners.processing.dvapi import DVTensor
+
+        start = time.perf_counter()
+        input_tensor = DVTensor(image.flatten(), self.input_param)
+        elapsed = time.perf_counter() - start
+        # Converting the tensor as part of the preprocess time.
+        self.timer.add_time("input", elapsed * 1e3)  # Convert to ms.
+
+        # Inference
+        with self.timer.time("inference"):
+            response = self.infer(input_tensor)
+
+        start = time.perf_counter()
+        output = dequantize_kinara(
+            response.get_output_tensors(),
+            method=self.parameters.nms)
+
+        outputs = []
+        for meta in self.outputs.metadata["outputs"]:
+            outputs.append(
+                output[meta["index"]].reshape(meta["shape"])
+            )
+        elapsed = time.perf_counter() - start
+
+        # Postprocessing
+        outputs = self.postprocessing(outputs)
+
+        # Fetching the tensor as part of the postprocess time.
+        self.timer.add_time("output", elapsed * 1e3)  # Convert to ms.
+
+        return outputs
+
+    def stop(self):
+        """
+        Close DVSession connection gracefully.
+        Attempts to close the session and suppresses any exceptions.
+        """
+        try:
+            self.conn.close()
+        except Exception:
+            try:
+                self.conn.__exit__(None, None, None)
+            except Exception:
+                pass
+
+    def get_input_type(self):
+        """
+        This returns the input type of the model. Kinara models
+        are always quantized in either INT8 or UINT8 datatypes.
+
+        Returns
+        -------
+        np.dtype
+            The input type of the model.
+        """
+        if self.preprocess_param.is_signed:
+            return np.int8
+        return np.uint8
+
+    def get_input_shape(self):
+        """
+        This fetches the model input shape. Kinara models are
+        always channels first.
+
+        Returns
+        -------
+        np.ndarray
+            The model input shape (batch size, channels, height, width).
+        """
+        input_shape = (self.input_param.batch_size,
+                       self.input_param.nch,
+                       self.input_param.height,
+                       self.input_param.width)
+        return input_shape

edgefirst/validator/runners/offline.py

@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+import os
+import json
+import warnings
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from edgefirst.validator.datasets.utils.transformations import (xcycwh2xyxy,
+                                                                xywh2xyxy)
+from edgefirst.validator.datasets.utils.fetch import get_shape
+from edgefirst.validator.runners.core import Runner
+
+if TYPE_CHECKING:
+    from edgefirst.validator.evaluators import ModelParameters, TimerContext
+
+
+class OfflineRunner(Runner):
+    """
+    This class reads model detection annotations stored in text files
+    that are in YOLO format. For more information on the YOLO format visit:
+    https://support.deepviewml.com/hc/en-us/articles/10869801702029
+
+    *Note: These text files should also include the model prediction scores
+    which adds to the YOLO format: [cls score xc yc width height]*
+
+    Use Case: PT models are ran using https://github.com/ultralytics/yolov5
+    repository. These model predictions will be stored in TXT files that
+    are in YOLO format. This class will read the text files to be validated.
+
+    Parameters
+    ----------
+    annotation_source: str
+        This is the path to the model prediction annotations
+        stored in text files with YOLO format annotations.
+        [cls score xc yc width height].
+    parameters: ModelParameters
+        These are the model parameters set from the command line.
+    annotation_extension: str
+        This represents the extension of the files that store
+        the prediction annotations. Only text files is supported
+        at the moment.
+    timer: TimerContext
+        A timer object for handling validation timings for the model.
+    """
+
+    def __init__(
+        self,
+        annotation_source: str,
+        parameters: ModelParameters,
+        timer: TimerContext,
+        annotation_extension='txt'
+    ):
+        super(OfflineRunner, self).__init__(
+            annotation_source, parameters, timer=timer)
+
+        self.annotation_extension = annotation_extension
+        if self.parameters.box_format not in ['xcycwh', 'xywh', 'xyxy']:
+            raise ValueError(
+                f"Unknown annotation format provided {self.parameters.box_format}.")
+
+        self.transformer = None
+        if self.parameters.box_format == 'xcycwh':
+            self.transformer = xcycwh2xyxy
+        elif self.parameters.box_format == 'xywh':
+            self.transformer = xywh2xyxy
+        else:
+            self.transformer = None
+
+        self.parameters.common.with_boxes = True
+        self.parameters.common.with_masks = False
+
+        self.__timings = {
+            'min_read_time': 0,
+            'max_read_time': 0,
+            'min_load_time': 0,
+            'max_load_time': 0,
+            'min_backbone_time': 0,
+            'max_backbone_time': 0,
+            'min_decode_time': 0,
+            'max_decode_time': 0,
+            'min_box_time': 0,
+            'max_box_time': 0,
+            'avg_read_time': 0,
+            'avg_load_time': 0,
+            'avg_backbone_time': 0,
+            'avg_decode_time': 0,
+            'avg_box_time': 0,
+        }
+        timings_path = os.path.join(annotation_source, "timings.json")
+        if os.path.exists(timings_path):
+            timings = {}
+            with open(timings_path) as file:
+                timings: dict = json.load(file)
+            self.parameters.engine = timings.get("engine", "cpu")
+            self.parameters.max_detections = timings.get("max_boxes", None)
+            self.parameters.iou_threshold = timings.get("iou", None)
+            self.parameters.score_threshold = timings.get("threshold", None)
+            self.parameters.common.shape = timings.get(
+                "shape", (1, 640, 640, 3))
+            self.parameters.common.dtype = timings.get("dtype", "float32")
+            self.parameters.warmup = None
+            self.__timings = timings.get("timings", {})
+        else:
+            # OFfline validation is not concerned with these parameters.
+            self.parameters.engine = "cpu"
+            self.parameters.nms = None
+            self.parameters.max_detections = None
+            self.parameters.iou_threshold = None
+            self.parameters.score_threshold = None
+            self.parameters.warmup = None
+            self.parameters.common.shape = (1, 640, 640, 3)
+            self.parameters.common.dtype = "float32"
+        timer.to_dict = self.get_timings
+
+        height, width = get_shape(self.parameters.common.shape)
+        try:
+            import edgefirst_python  # type: ignore
+            self.parameters.common.input_dst = edgefirst_python.TensorImage(
+                width, height, edgefirst_python.FourCC.RGBA
+            )
+        except ImportError:
+            if self.parameters.common.backend == "hal":
+                raise ImportError(
+                    "EdgeFirst HAL is needed to create a TensorImage destination."
+                )
+
+    def run_single_instance(self, image: str) -> Any:
+        """
+        This method reads one prediction annotation file based on the
+        image name and returns the bounding boxes and labels.
+
+        Parameters
+        ----------
+        image: str
+            The path to the image. This is used to match the
+            annotation to be read.
+
+        Returns
+        -------
+        Any:
+            This could either return detection outputs after NMS.
+                np.ndarray
+                    The prediction bounding boxes.. [[box1], [box2], ...].
+                np.ndarray
+                    The prediction labels.. [cl1, cl2, ...].
+                np.ndarray
+                    The prediction confidence scores.. [score, score, ...]
+                    normalized between 0 and 1.
+            This could also return segmentation masks.
+                np.ndarray
+        """
+        annotation_path = os.path.join(self.model, "{}.{}".format(
+            os.path.splitext(os.path.basename(image))[0],
+            self.annotation_extension
+        ))
+
+        try:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                annotation = np.genfromtxt(annotation_path)
+        except FileNotFoundError:
+            return np.array([], dtype=np.float32), \
+                np.array([], dtype=np.int32), np.array([], dtype=np.float32)
+
+        if len(annotation):
+            annotation = annotation.reshape(-1, 6)
+            boxes = annotation[:, 2:6]
+            boxes = self.transformer(boxes) if self.transformer else boxes
+        else:
+            return np.array([], dtype=np.float32), \
+                np.array([], dtype=np.int32), np.array([], dtype=np.float32)
+
+        scores = annotation[:, 1:2].flatten().astype(np.float32)
+        labels = annotation[:, 0:1].flatten().astype(
+            np.int32) + self.parameters.label_offset
+
+        return boxes, labels, scores
+
+    def get_timings(self) -> dict:
+        """
+        Returns a summary of all the timings:
+        (mean, avg, max) of (load, inference, box).
+
+        Returns
+        -------
+        dict
+            The timings in milliseconds.
+
+            .. code-block:: python
+
+                {
+                    'min_read_time': minimum time to read the input,
+                    'max_read_time': maximum time to read the input,
+                    'min_load_time': minimum time to preprocess the input,
+                    'max_load_time': maximum time to preprocess the input,
+                    'min_backbone_time': minimum time to run the model,
+                    'max_backbone_time': maximum time to run the model,
+                    'min_decode_time': minimum time to decode the outputs,
+                    'max_decode_time': maximum time to decode the outputs,
+                    'min_box_time': minimum time to process the outputs,
+                    'max_box_time': maximum time to process the outputs,
+                    'avg_read_time': average time to read the input,
+                    'avg_load_time': average time to preprocess the input,
+                    'avg_backbone_time': average time to run the model,
+                    'avg_decode_time': average time to decode the outputs,
+                    'avg_box_time': average time to process the outputs,
+                }
+        """
+        # Convert timings to ms.
+        return {
+            'min_read_time': self.__timings.get("min_read_time", 0) * 1e3,
+            'max_read_time': self.__timings.get("max_read_time", 0) * 1e3,
+            'min_load_time': self.__timings.get("min_load_time", 0) * 1e3,
+            'max_load_time': self.__timings.get("max_load_time", 0) * 1e3,
+            'min_backbone_time': self.__timings.get("min_backbone_time", 0) * 1e3,
+            'max_backbone_time': self.__timings.get("max_backbone_time", 0) * 1e3,
+            'min_decode_time': self.__timings.get("min_decode_time", 0) * 1e3,
+            'max_decode_time': self.__timings.get("max_decode_time", 0) * 1e3,
+            'min_box_time': self.__timings.get("min_box_time", 0) * 1e3,
+            'max_box_time': self.__timings.get("max_box_time", 0) * 1e3,
+            'avg_read_time': self.__timings.get("avg_read_time", 0) * 1e3,
+            'avg_load_time': self.__timings.get("avg_load_time", 0) * 1e3,
+            'avg_backbone_time': self.__timings.get("avg_backbone_time", 0) * 1e3,
+            'avg_decode_time': self.__timings.get("avg_decode_time", 0) * 1e3,
+            'avg_box_time': self.__timings.get("avg_box_time", 0) * 1e3,
+        }