edgefirst-validator 4.2.1__py3-none-any.whl

edgefirst/validator/evaluators/callbacks/studio.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import traceback
+from typing import TYPE_CHECKING
+
+import threading
+from queue import Queue, Empty
+import time
+
+from edgefirst.validator.evaluators.callbacks import PlotsCallback, CallbacksList
+
+if TYPE_CHECKING:
+    from edgefirst.validator.publishers import StudioPublisher
+    from edgefirst.validator.evaluators import Evaluator
+
+
+class StudioProgress:
+    """
+    Deploy standard validation from the existing evaluator objects
+    but also provide communication to EdgeFirst Studio to report
+    the progress and the final metrics and evaluation of the model
+    performance.
+
+    Parameters
+    ----------
+    evaluator: Evaluator
+        This object handles running validation by iterating through
+        the dataset samples and run model inference to calculate the
+        validation metrics at the end of the process.
+    studio_publisher: StudioPublisher
+        Handles the state updates to the validation session in EdgeFirst Studio.
+    stage: str
+        The current stage to update for the progress in Studio.
+    """
+
+    def __init__(
+        self,
+        evaluator: Evaluator,
+        studio_publisher: StudioPublisher,
+        stage: str = "validate"
+    ):
+        self.evaluator = evaluator
+        plots_callback = PlotsCallback(studio_publisher=studio_publisher,
+                                       parameters=self.evaluator.parameters,
+                                       stage=stage)
+        self.callbacks = CallbacksList([plots_callback])
+
+        # Non-blocking queue + single background worker thread to execute callbacks
+        # Bounded queue prevents unbounded memory growth;
+        # if full we fallback to sync execution.
+        self._queue: Queue = Queue(maxsize=1000)
+        self._stop_event = threading.Event()
+        self._last_update = 0
+        # NOTE: Increasing this interval results in the functions not being
+        # called.
+        self._update_interval = 0.0001  # Minimum seconds between updates.
+        self._worker = threading.Thread(target=self._worker_loop,
+                                        name="studio-callback-worker",
+                                        daemon=True)
+        self._worker.start()
+
+    def _worker_loop(self):
+        """
+        Background worker that executes
+        callback functions from the queue.
+        """
+        while True:
+            try:
+                fn, kwargs = self._queue.get(timeout=0.5)
+            except Empty:
+                if self._stop_event.is_set() and self._queue.empty():
+                    break
+                continue
+            try:
+                try:
+                    fn(**kwargs)
+                except Exception:
+                    traceback.print_exc()
+            finally:
+                try:
+                    self._queue.task_done()
+                except Exception:
+                    pass
+
+    def _dispatch(self, method_name: str, /, **kwargs):
+        """
+        Enqueue a callback method for asynchronous execution.
+        If the queue is full, execute synchronously as a best-effort
+        fallback to avoid dropping important updates.
+
+        Parameters
+        ----------
+        method_name: str
+            The name of the callback method to invoke.
+        **kwargs: dict
+            The keyword arguments to pass to the callback method.
+        """
+        fn = getattr(self.callbacks, method_name)
+        # Always process begin/end/error events
+        critical = method_name in ('on_test_begin',
+                                   'on_test_end',
+                                   'on_test_error')
+        current_time = time.monotonic()
+        if critical or (current_time -
+                        self._last_update) >= self._update_interval:
+            try:
+                self._queue.put_nowait((fn, kwargs))
+                self._last_update = current_time
+            except Exception:
+                if critical:
+                    # Only fall back to sync for critical updates.
+                    try:
+                        fn(**kwargs)
+                    except Exception:
+                        traceback.print_exc()
+
+    def _drain_and_stop(self, timeout: float = 10.0):
+        """
+        Wait for queued tasks to finish
+        and stop the background worker.
+
+        Parameters
+        ----------
+        timeout: float
+            The maximum time to wait for the worker to stop.
+        """
+        # Wait for tasks to be processed
+        try:
+            self._queue.join()
+        except Exception:
+            pass
+        self._stop_event.set()
+        self._worker.join(timeout=timeout)
+
+    def group_evaluation(self, epoch: int = 0, reset: bool = True):
+        """
+        Runs model validation on all samples in the dataset.
+
+        Parameters
+        ----------
+        epoch: int
+            This is the training epoch number. This
+            parameter is internal for ModelPack usage.
+            Standalone validation does not use this parameter.
+        reset: bool
+            This is an optional parameter that controls the reset state.
+            By default, it will reset at the end of validation to erase
+            the data in the containers.
+        """
+        save_image = bool(self.evaluator.parameters.validation.visualize or
+                          self.evaluator.parameters.validation.tensorboard)
+
+        logs = {
+            "total": len(self.evaluator.dataset)
+        }
+
+        i = 0
+        try:
+            self._dispatch("on_test_begin", logs=logs)
+            for instance in self.evaluator.instance_collector():
+                i += 1
+                self._dispatch("on_test_batch_begin", step=i, logs=logs)
+                if self.evaluator.parameters.validation.display >= 0:
+                    if (self.evaluator.counter <
+                            self.evaluator.parameters.validation.display):
+                        save_image = True
+                        self.evaluator.counter += 1
+                    else:
+                        save_image = False
+
+                self.evaluator.single_evaluation(
+                    instance, epoch=epoch, save_image=save_image)
+                self._dispatch("on_test_batch_end", step=i, logs=logs)
+            metrics, plots = self.end(epoch=epoch, reset=reset)
+
+            if (self.evaluator.parameters.model.common.with_boxes and
+                    self.evaluator.parameters.model.common.with_masks):
+                logs["multitask"] = metrics
+                logs["plots"] = plots
+                logs["timings"] = metrics.timings
+
+            elif self.evaluator.parameters.model.common.with_boxes:
+                logs["detection"] = metrics
+                logs["plots"] = plots
+                logs["timings"] = metrics.timings
+
+            elif self.evaluator.parameters.model.common.with_masks:
+                logs["segmentation"] = metrics
+                logs["plots"] = plots
+                logs["timings"] = metrics.timings
+
+            self._dispatch("on_test_end", logs=logs)
+            # Ensure queued callback tasks complete before returning from
+            # evaluation
+            self._drain_and_stop()
+
+        except Exception as e:
+            self._dispatch("on_test_error", step=i, error=e, logs=logs)
+            # Ensure error callback is processed and worker stopped
+            self._drain_and_stop()
+            error = traceback.format_exc()
+            print(error)
+            raise e
+
+    def end(self, epoch: int = 0, reset: bool = True):
+        """
+        Calculate final metrics and publish the results into
+        EdgeFirst Studio.
+
+        Parameters
+        ----------
+        epoch: int
+            This is the training epoch number. This
+            parameter is internal for ModelPack usage.
+            Standalone validation does not use this parameter.
+        reset: bool
+            This is an optional parameter that controls the reset state.
+            By default, it will reset at the end of validation to erase
+            the data in the containers.
+
+        Returns
+        -------
+        Tuple[Metrics, Metrics]
+            This returns the detection and segmentation metrics for
+            multitask. Otherwise, for single tasks, only one or
+            the other is returned.
+        """
+        # Ensure any pending callback tasks are flushed
+        # before final metrics are posted
+        try:
+            self._queue.join()
+        except Exception:
+            pass
+        return self.evaluator.end(epoch=epoch, reset=reset)

edgefirst/validator/evaluators/core.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+
+import os
+from copy import deepcopy
+from typing import TYPE_CHECKING, Tuple, List
+
+import numpy as np
+import matplotlib.figure
+
+from edgefirst.validator.publishers.utils.logger import logger
+from edgefirst.validator.visualize.utils.plots import close_figures
+from edgefirst.validator.datasets.utils.transformations import labels2string
+from edgefirst.validator.publishers import ConsolePublisher, TensorBoardPublisher
+
+if TYPE_CHECKING:
+    from edgefirst.validator.evaluators import CombinedParameters
+    from edgefirst.validator.datasets import Dataset
+    from edgefirst.validator.metrics import Metrics
+    from edgefirst.validator.runners import Runner
+    from edgefirst.validator.metrics import Plots
+
+
+class Evaluator:
+    """
+    Abstract class that provides a template for the
+    Evaluators (detection or segmentation).
+
+    Parameters
+    ----------
+    parameters: CombinedParameters
+        This is a container for the model, dataset, and validation parameters
+    runner: Runner
+        This object provides methods to run inference on the model provided.
+    dataset: Dataset
+        A type of dataset object responsible for reading different types
+        of datasets such as Darknet, TFRecords, or EdgeFirst Datasets.
+    """
+
+    def __init__(
+        self,
+        parameters: CombinedParameters,
+        runner: Runner,
+        dataset: Dataset,
+    ):
+        self.parameters = parameters
+        self.runner = runner
+        self.dataset = dataset
+
+        self.console_writer = ConsolePublisher(
+            self.parameters.validation.visualize)
+        self.tensorboard_writer = None
+        if self.parameters.validation.tensorboard:
+            self.tensorboard_writer = TensorBoardPublisher(
+                self.parameters.validation.tensorboard)
+
+        self.model_name = os.path.basename(os.path.normpath(
+            self.parameters.model.model_path))
+        if os.path.isfile(self.parameters.dataset.dataset_path):
+            self.dataset_name = os.path.basename(os.path.normpath(
+                os.path.dirname(self.parameters.dataset.dataset_path)))
+        else:
+            self.dataset_name = os.path.basename(os.path.normpath(
+                self.parameters.dataset.dataset_path))
+
+        if self.parameters.validation.tensorboard:
+            self.save_path = self.parameters.validation.tensorboard
+        elif self.tensorboard_writer:
+            self.save_path = self.tensorboard_writer.logdir
+        elif self.parameters.validation.visualize:
+            self.save_path = self.parameters.validation.visualize
+        else:
+            self.save_path = None
+
+        self.metrics = None
+        self.confusion_matrix = None
+
+        # This counter is used to determine the number of images saved.
+        self.counter = 0
+
+    def instance_collector(self):
+        """Abstract Method"""
+        raise NotImplementedError("This is an abstract method")
+
+    def evaluate(self, instance: dict):
+        """Abstract Method"""
+        pass
+
+    def single_evaluation(self, instance: dict, epoch: int, save_image: bool):
+        """
+        Run model evaluation on a single image/sample.
+
+        Parameters
+        ----------
+        instance: dict
+            This contains the ground truth and model prediction instances
+            with keys "gt_instance", "dt_instance".
+        epoch: int
+            This is the training epoch number. This
+            parameter is internal for ModelPack usage.
+            Standalone validation does not use this parameter.
+        save_image: bool
+            If set to True, this will save the image
+            with drawn bounding box results.
+        """
+        self.evaluate(instance=instance)
+
+        if save_image:
+            gt_instance = instance.get("gt_instance", None)
+            dt_instance = instance.get("dt_instance", None)
+
+            # Convert labels from integers to string for detection.
+            gt_instance.labels = np.array(labels2string(
+                gt_instance.labels, self.parameters.dataset.labels))
+            dt_instance.labels = np.array(labels2string(
+                dt_instance.labels, self.parameters.dataset.labels))
+            self.visualize(gt_instance, dt_instance, epoch=epoch)
+
+    def group_evaluation(self, epoch: int = 0, reset: bool = True):
+        """
+        Runs model validation on all samples in the dataset.
+
+        Parameters
+        ----------
+        epoch: int
+            This is the training epoch number. This
+            parameter is internal for ModelPack usage.
+            Standalone validation does not use this parameter.
+        reset: bool
+            This is an optional parameter that controls the reset state.
+            By default, it will reset at the end of validation to erase
+            the data in the containers.
+        """
+        save_image = bool(self.parameters.validation.visualize or
+                          self.parameters.validation.tensorboard)
+
+        for instance in self.instance_collector():
+            if self.parameters.validation.display >= 0:
+                if self.counter < self.parameters.validation.display:
+                    save_image = True
+                    self.counter += 1
+                else:
+                    save_image = False
+
+            if instance.get("dt_instance", None) is None:
+                logger(
+                    "VisionPack Trial Expired. Please use a licensed version" +
+                    " for complete validation. Contact support@au-zone.com" +
+                    " for more information.", code="WARNING")
+                break
+
+            self.single_evaluation(
+                instance=instance, epoch=epoch, save_image=save_image)
+        return self.end(epoch=epoch, reset=reset)
+
+    def visualize(self):
+        """Absract Method"""
+        raise NotImplementedError("This is an abstract method")
+
+    def end(
+        self,
+        epoch: int = 0,
+        reset: bool = True,
+        publish: bool = True
+    ) -> Tuple[Metrics, Plots]:
+        """
+        Computes the final metrics and generates the validation plots
+        to save the results in disk or publishes to Tensorboard.
+
+        Parameters
+        ----------
+        epoch: int
+            This is the training epoch number. This
+            parameter is internal for ModelPack usage.
+            Standalone validation does not use this parameter.
+        reset: bool
+            This is an optional parameter that controls the reset state.
+            By default, it will reset at the end of validation to erase
+            the data in the containers.
+        publish: bool
+            Specify to publish and print the metrics. Default to True.
+
+        Returns
+        -------
+        metrics: Metrics
+            This is a container for the detection metrics.
+        plots: Plots
+            This is a container for the validation data for plotting.
+        """
+        if hasattr(self.runner, "stop"):
+            self.runner.stop()
+
+        if self.runner:
+            self.metrics.metrics.timings = self.runner.timer.to_dict()
+        self.metrics.run_metrics()
+
+        # Plot Operations
+        if self.parameters.validation.plots:
+            if self.parameters.model.common.with_boxes:
+                self.metrics.plots.curve_labels = labels2string(
+                    self.metrics.plots.curve_labels, self.parameters.dataset.labels)
+                if (self.parameters.validation.method in ["ultralytics", "yolov7"]
+                        and self.confusion_matrix is not None):
+                    self.metrics.plots.confusion_matrix = self.confusion_matrix.matrix
+
+            if self.parameters.validation.visualize or self.tensorboard_writer:
+                plots = self.get_plots()
+
+                if self.parameters.validation.visualize:
+                    self.save_plots(plots)
+                elif self.tensorboard_writer:
+                    self.publish_plots(plots, epoch)
+                close_figures(plots)
+
+        if publish:
+            # Metric Operations
+            if self.tensorboard_writer:
+                self.tensorboard_writer.publish_metrics(
+                    metrics=self.metrics.metrics,
+                    parameters=self.parameters,
+                    step=epoch,
+                )
+            else:
+                table = self.console_writer(metrics=self.metrics.metrics,
+                                            parameters=self.parameters)
+                if self.parameters.validation.visualize:
+                    self.console_writer.save_metrics(table)
+
+            if self.parameters.validation.csv_out:
+                self.console_writer.save_csv_metrics(
+                    metrics=self.metrics.metrics, parameters=self.parameters)
+
+        # Prevent the reset from taking effect.
+        metrics = deepcopy(self.metrics.metrics)
+        plots = deepcopy(self.metrics.plots)
+        if reset:
+            self.metrics.reset()
+        return metrics, plots
+
+    def stop(self):
+        """
+        Stops any active running processes.
+        """
+        if hasattr(self.runner, "stop"):
+            self.runner.stop()
+
+    def get_plots(self) -> List[matplotlib.figure.Figure]:
+        """Absract Method"""
+        pass
+
+    def save_plots(self, plots: List[matplotlib.figure.Figure]):
+        """Absract Method"""
+        pass
+
+    def publish_plots(
+            self, plots: List[matplotlib.figure.Figure], epoch: int = 0):
+        """Absract Method"""
+        pass