edgefirst-validator 4.2.1__py3-none-any.whl

edgefirst/validator/evaluators/__init__.py

@@ -0,0 +1,17 @@
+from edgefirst.validator.evaluators.callbacks import (Callback, CallbacksList,
+                                                      PlotsCallback,
+                                                      StudioProgress)
+from edgefirst.validator.evaluators.parameters import (Parameters,
+                                                       CommonParameters,
+                                                       CombinedParameters,
+                                                       ModelParameters,
+                                                       DatasetParameters,
+                                                       ValidationParameters)
+from edgefirst.validator.evaluators.utils import (Matcher, DetectionClassifier,
+                                                  TimerContext)
+from edgefirst.validator.evaluators.core import Evaluator
+from edgefirst.validator.evaluators.detection import (YOLOValidator,
+                                                      EdgeFirstValidator)
+from edgefirst.validator.evaluators.segmentation import (YOLOSegmentationValidator,
+                                                         SegmentationValidator)
+from edgefirst.validator.evaluators.multitask import MultitaskValidator

edgefirst/validator/evaluators/callbacks/__init__.py

@@ -0,0 +1,3 @@
+from edgefirst.validator.evaluators.callbacks.core import Callback, CallbacksList
+from edgefirst.validator.evaluators.callbacks.plots import PlotsCallback
+from edgefirst.validator.evaluators.callbacks.studio import StudioProgress

edgefirst/validator/evaluators/callbacks/core.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List
+
+if TYPE_CHECKING:
+    from edgefirst.validator.publishers import StudioPublisher
+    from edgefirst.validator.evaluators import CombinedParameters
+
+
+class Callback(object):
+    """
+    A class to handle callback functions during validation.
+    Callbacks are used to communicate the progress of the stages
+    to EdgeFirst Studio.
+
+    Parameters
+    -----------
+    studio_publisher: StudioPublisher
+        Publishes metrics, timings, plots, and
+        progress to EdgeFirst Studio.
+    parameters: CombinedParameters
+        These are the model, dataset, and validation parameters
+        set from the command line.
+    stage: str
+        The current stage to update for the progress in Studio.
+    """
+
+    def __init__(
+        self,
+        studio_publisher: StudioPublisher,
+        parameters: CombinedParameters,
+        stage: str = "validate"
+    ):
+        self.studio_publisher = studio_publisher
+        self.parameters = parameters
+        self.message = 'Running Validation'
+        self.stage = stage
+
+    def on_test_begin(self, logs=None):
+        """
+        Called at the beginning of the test.
+
+        Parameters
+        ----------
+        logs: dict
+            A dictionary containing logs related to the test (default is None).
+        """
+        return
+
+    def on_test_batch_begin(self, step: int, logs=None):
+        """
+        Called at the beginning of each test batch.
+
+        Parameters
+        ----------
+        step : int
+            The index of the current test batch.
+        logs : dict, optional
+            A dictionary containing logs related
+            to the test batch (default is None).
+        """
+        return
+
+    def on_test_batch_end(self, step: int, logs=None):
+        """
+        Called at the end of each test batch.
+
+        Parameters
+        ----------
+        step : int
+            The index of the current test batch.
+        logs : dict, optional
+            A dictionary containing logs related
+            to the test batch (default is None).
+        """
+        return
+
+    def on_test_error(self, step: int, error, logs=None):
+        """
+        Called when the callback raises an error.
+
+        Parameters
+        ----------
+        step : int
+            The index of the current test batch.
+        error: Exception
+            The error exception.
+        logs : dict, optional
+            A dictionary containing logs related
+            to the test batch (default is None).
+        """
+        return
+
+    def on_test_end(self, logs=None):
+        """
+        Called at the end of the test.
+
+        Parameters
+        ----------
+        logs : dict, optional
+            A dictionary containing logs related to the test (default is None).
+        """
+        return
+
+
+class CallbacksList(object):
+    """
+    A list of callbacks that can be triggered during the validation process.
+
+    Parameters
+    ----------
+    callbacks : List[Callback], optional
+        A list of callback objects to be triggered
+        during the testing process (default is None).
+    """
+
+    def __init__(
+        self,
+        callbacks: List[Callback] = None
+    ):
+        self.callbacks = callbacks if callbacks else []
+
+    def on_test_begin(self, logs=None):
+        """
+        Triggers the `on_test_begin` method of each callback in the list.
+
+        Parameters
+        ----------
+        logs: dict
+            A dictionary containing logs related to the test (default is None).
+        """
+        for callback in self.callbacks:
+            callback.on_test_begin(logs)
+
+    def on_test_batch_begin(self, step: int, logs=None):
+        """
+        Triggers the `on_test_batch_begin` method of each callback in the list.
+
+        Parameters
+        ----------
+        step : int
+            The index of the current test batch.
+        logs : dict, optional
+            A dictionary containing logs related
+            to the test batch (default is None).
+        """
+        for callback in self.callbacks:
+            callback.on_test_batch_begin(step, logs)
+
+    def on_test_batch_end(self, step: int, logs=None):
+        """
+        Triggers the `on_test_batch_end` method of each callback in the list.
+
+        Parameters
+        ----------
+        step : int
+            The index of the current test batch.
+        logs : dict, optional
+            A dictionary containing logs related
+            to the test batch (default is None).
+        """
+        for callback in self.callbacks:
+            callback.on_test_batch_end(step, logs)
+
+    def on_test_error(self, step: int, error, logs=None):
+        """
+        Triggers the `on_test_error` method of each callback in the list.
+
+        Parameters
+        ----------
+        step : int
+            The index of the current test batch.
+        error: Exception
+            The error exception.
+        logs : dict, optional
+            A dictionary containing logs related
+            to the test batch (default is None).
+        """
+        for callback in self.callbacks:
+            callback.on_test_error(step, error, logs)
+
+    def on_test_end(self, logs=None):
+        """
+        Triggers the `on_test_end` method of each callback in the list.
+
+        Parameters
+        ----------
+        logs : dict, optional
+            A dictionary containing logs related to the test (default is None).
+        """
+        for callback in self.callbacks:
+            callback.on_test_end(logs)