scale-nucleus 0.1.24__py3-none-any.whl → 0.6.4__py3-none-any.whl

nucleus/validate/eval_functions/available_eval_functions.py

@@ -0,0 +1,212 @@
+import itertools
+from typing import Callable, Dict, List, Type, Union
+
+from nucleus.logger import logger
+from nucleus.validate.eval_functions.base_eval_function import BaseEvalFunction
+
+from ..data_transfer_objects.eval_function import EvalFunctionEntry
+from ..errors import EvalFunctionNotAvailableError
+
+MEAN_AVG_PRECISION_NAME = "mean_average_precision_boxes"
+
+
+class BoundingBoxIOU(BaseEvalFunction):
+    @classmethod
+    def expected_name(cls) -> str:
+        return "bbox_iou"
+
+
+class BoundingBoxMeanAveragePrecision(BaseEvalFunction):
+    @classmethod
+    def expected_name(cls) -> str:
+        return "bbox_map"
+
+
+class BoundingBoxRecall(BaseEvalFunction):
+    @classmethod
+    def expected_name(cls) -> str:
+        return "bbox_recall"
+
+
+class BoundingBoxPrecision(BaseEvalFunction):
+    @classmethod
+    def expected_name(cls) -> str:
+        return "bbox_precision"
+
+
+class CategorizationF1(BaseEvalFunction):
+    @classmethod
+    def expected_name(cls) -> str:
+        return "cat_f1"
+
+
+class CustomEvalFunction(BaseEvalFunction):
+    @classmethod
+    def expected_name(cls) -> str:
+        raise NotImplementedError(
+            "Custm evaluation functions are coming soon"
+        )  # Placeholder: See super().eval_func_entry for actual name
+
+
+class StandardEvalFunction(BaseEvalFunction):
+    """Class for standard Model CI eval functions that have not been added as attributes on
+    AvailableEvalFunctions yet.
+    """
+
+    def __init__(self, eval_function_entry: EvalFunctionEntry):
+        logger.warning(
+            "Standard function %s not implemented as an attribute on AvailableEvalFunctions",
+            eval_function_entry.name,
+        )
+        super().__init__(eval_function_entry)
+
+    @classmethod
+    def expected_name(cls) -> str:
+        return "public_function"  # Placeholder: See super().eval_func_entry for actual name
+
+
+class EvalFunctionNotAvailable(BaseEvalFunction):
+    def __init__(
+        self, not_available_name: str
+    ):  # pylint: disable=super-init-not-called
+        self.not_available_name = not_available_name
+
+    def __call__(self, *args, **kwargs):
+        self._raise_error()
+
+    def _op_to_test_metric(self, *args, **kwargs):
+        self._raise_error()
+
+    def _raise_error(self):
+        raise EvalFunctionNotAvailableError(
+            f"Eval function '{self.not_available_name}' is not available to the current user. "
+            f"Is Model CI enabled for the user?"
+        )
+
+    @classmethod
+    def expected_name(cls) -> str:
+        return "public_function"  # Placeholder: See super().eval_func_entry for actual name
+
+
+EvalFunction = Union[
+    Type[BoundingBoxIOU],
+    Type[BoundingBoxMeanAveragePrecision],
+    Type[BoundingBoxPrecision],
+    Type[BoundingBoxRecall],
+    Type[CustomEvalFunction],
+    Type[EvalFunctionNotAvailable],
+    Type[StandardEvalFunction],
+]
+
+
+class AvailableEvalFunctions:
+    """Collection class that acts as a common entrypoint to access evaluation functions. Standard evaluation functions
+    provided by Scale are attributes of this class.
+
+    The available evaluation functions are listed in the sample below::
+
+        e = client.validate.eval_functions
+        unit_test_criteria = [
+            e.bbox_iou() > 5,
+            e.bbox_map() > 0.95,
+            e.bbox_precision() > 0.8,
+            e.bbox_recall() > 0.5,
+        ]
+    """
+
+    # pylint: disable=too-many-instance-attributes
+
+    def __init__(self, available_functions: List[EvalFunctionEntry]):
+        assert (
+            available_functions
+        ), "Passed no available functions for current user. Is the feature flag enabled?"
+        self._public_func_entries: Dict[str, EvalFunctionEntry] = {
+            f.name: f for f in available_functions if f.is_public
+        }
+        # NOTE: Public are assigned
+        self._public_to_function: Dict[str, BaseEvalFunction] = {}
+        self._custom_to_function: Dict[str, CustomEvalFunction] = {
+            f.name: CustomEvalFunction(f)
+            for f in available_functions
+            if not f.is_public
+        }
+        self.bbox_iou = self._assign_eval_function_if_defined(BoundingBoxIOU)  # type: ignore
+        self.bbox_precision = self._assign_eval_function_if_defined(
+            BoundingBoxPrecision  # type: ignore
+        )
+        self.bbox_recall = self._assign_eval_function_if_defined(
+            BoundingBoxRecall  # type: ignore
+        )
+        self.bbox_map = self._assign_eval_function_if_defined(
+            BoundingBoxMeanAveragePrecision  # type: ignore
+        )
+        self.cat_f1 = self._assign_eval_function_if_defined(
+            CategorizationF1  # type: ignore
+        )
+
+        # Add public entries that have not been implemented as an attribute on this class
+        for func_entry in self._public_func_entries.values():
+            if func_entry.name not in self._public_to_function:
+                self._public_to_function[
+                    func_entry.name
+                ] = StandardEvalFunction(func_entry)
+
+    def __repr__(self):
+        """Standard functions are ones Scale provides and custom ones customer defined"""
+        # NOTE: setting to lower to be consistent with attribute names
+        functions_lower = [
+            str(name).lower() for name in self._public_func_entries.keys()
+        ]
+        return (
+            f"<AvailableEvaluationFunctions: public:{functions_lower} "
+            f"private: {list(self._custom_to_function.keys())}"
+        )
+
+    @property
+    def public_functions(self) -> Dict[str, BaseEvalFunction]:
+        """Standard functions provided by Model CI.
+
+        Notes:
+            These functions are also available as attributes on :class:`AvailableEvalFunctions`
+
+        Returns:
+            Dict of function name to :class:`BaseEvalFunction`.
+        """
+        return self._public_to_function
+
+    @property
+    def private_functions(self) -> Dict[str, CustomEvalFunction]:
+        """Custom functions uploaded to Model CI
+
+        Returns:
+            Dict of function name to :class:`CustomEvalFunction`.
+        """
+        return self._custom_to_function
+
+    def _assign_eval_function_if_defined(
+        self,
+        eval_function_constructor: Callable[[EvalFunctionEntry], EvalFunction],
+    ):
+        """Helper function for book-keeping and assignment of standard Scale provided functions that are accessible
+        via attribute access
+        """
+        # TODO(gunnar): Too convoluted .. simplify
+        expected_name = eval_function_constructor.expected_name()  # type: ignore
+        if expected_name in self._public_func_entries:
+            definition = self._public_func_entries[expected_name]
+            eval_function = eval_function_constructor(definition)
+            self._public_to_function[expected_name] = eval_function  # type: ignore
+            return eval_function
+        else:
+            return EvalFunctionNotAvailable(expected_name)
+
+    def from_id(self, eval_function_id: str):
+        for eval_func in itertools.chain(
+            self._public_to_function.values(),
+            self._custom_to_function.values(),
+        ):
+            if eval_func.id == eval_function_id:
+                return eval_func
+        raise EvalFunctionNotAvailableError(
+            f"Could not find Eval Function with id {eval_function_id}"
+        )

nucleus/validate/eval_functions/base_eval_function.py

@@ -0,0 +1,60 @@
+import abc
+
+from ..constants import ThresholdComparison
+from ..data_transfer_objects.eval_function import (
+    EvalFunctionEntry,
+    EvaluationCriterion,
+)
+
+
+class BaseEvalFunction(abc.ABC):
+    """Abstract base class for concrete implementations of EvalFunctions
+
+    Operating on this class with comparison operators produces an EvaluationCriterion
+    """
+
+    def __init__(self, eval_func_entry: EvalFunctionEntry):
+        self.eval_func_entry = eval_func_entry
+        self.id = eval_func_entry.id
+        self.name = eval_func_entry.name
+
+    def __repr__(self):
+        return f"<EvalFunction: name={self.name}, id={self.id}>"
+
+    @classmethod
+    @abc.abstractmethod
+    def expected_name(cls) -> str:
+        """Name to look for in the EvalFunctionDefinitions"""
+
+    def __call__(self) -> "BaseEvalFunction":
+        """Adding call to prepare for being able to pass parameters to function
+
+        Notes:
+            Technically now you could do something like eval_function > 0.5 but we want it
+            to look like eval_function() > 0.5 to support eval_function(parameters) > 0.5
+            in the future
+        """
+        return self
+
+    def __gt__(self, other) -> EvaluationCriterion:
+        return self._op_to_test_metric(ThresholdComparison.GREATER_THAN, other)
+
+    def __ge__(self, other) -> EvaluationCriterion:
+        return self._op_to_test_metric(
+            ThresholdComparison.GREATER_THAN_EQUAL_TO, other
+        )
+
+    def __lt__(self, other) -> EvaluationCriterion:
+        return self._op_to_test_metric(ThresholdComparison.LESS_THAN, other)
+
+    def __le__(self, other) -> EvaluationCriterion:
+        return self._op_to_test_metric(
+            ThresholdComparison.LESS_THAN_EQUAL_TO, other
+        )
+
+    def _op_to_test_metric(self, comparison: ThresholdComparison, value):
+        return EvaluationCriterion(
+            eval_function_id=self.eval_func_entry.id,
+            threshold_comparison=comparison,
+            threshold=value,
+        )

nucleus/validate/scenario_test.py

@@ -0,0 +1,143 @@
+"""Scenario Tests combine collections of data and evaluation metrics to accelerate model evaluation.
+
+With Model CI Scenario Tests, an ML engineer can define a Scenario Test from critical
+edge case scenarios that the model must get right (e.g. pedestrians at night),
+and have confidence that they’re always shipping the best model.
+"""
+from dataclasses import dataclass, field
+from typing import List
+
+from ..connection import Connection
+from ..constants import NAME_KEY, SLICE_ID_KEY
+from ..dataset_item import DatasetItem
+from .data_transfer_objects.eval_function import EvaluationCriterion
+from .data_transfer_objects.scenario_test_evaluations import GetEvalHistory
+from .data_transfer_objects.scenario_test_metric import AddScenarioTestMetric
+from .scenario_test_evaluation import ScenarioTestEvaluation
+from .scenario_test_metric import ScenarioTestMetric
+
+DATASET_ITEMS_KEY = "dataset_items"
+
+
+@dataclass
+class ScenarioTest:
+    """A Scenario Test combines a slice and at least one evaluation criterion. A :class:`ScenarioTest` is not created through
+    the default constructor but using the instructions shown in :class:`Validate`. This :class:`ScenarioTest` class only
+    simplifies the interaction with the scenario tests from this SDK.
+
+    Attributes:
+        id (str): The ID of the scenario test.
+        connection (Connection): The connection to Nucleus API.
+        name (str): The name of the scenario test.
+        slice_id (str): The ID of the associated Nucleus slice.
+    """
+
+    id: str
+    connection: Connection = field(repr=False)
+    name: str = field(init=False)
+    slice_id: str = field(init=False)
+
+    def __post_init__(self):
+        # TODO(gunnar): Remove this pattern. It's too slow. We should get all the info required in one call
+        response = self.connection.get(
+            f"validate/scenario_test/{self.id}/info",
+        )
+        self.name = response[NAME_KEY]
+        self.slice_id = response[SLICE_ID_KEY]
+
+    def add_criterion(
+        self, evaluation_criterion: EvaluationCriterion
+    ) -> ScenarioTestMetric:
+        """Creates and adds a new criteria to the :class:`ScenarioTest`. ::
+
+            import nucleus
+            client = nucleus.NucleusClient("YOUR_SCALE_API_KEY")
+            scenario_test = client.validate.create_scenario_test(
+                "sample_scenario_test", "slc_bx86ea222a6g057x4380"
+            )
+
+            e = client.validate.eval_functions
+            # Assuming a user would like to add all available public evaluation functions as criteria
+            scenario_test.add_criterion(
+                e.bbox_iou() > 0.5
+            )
+            scenario_test.add_criterion(
+                e.bbox_map() > 0.85
+            )
+            scenario_test.add_criterion(
+                e.bbox_precision() > 0.7
+            )
+            scenario_test.add_criterion(
+                e.bbox_recall() > 0.6
+            )
+
+        Args:
+            evaluation_criterion: :class:`EvaluationCriterion` created by comparison with an :class:`EvalFunction`
+
+        Returns:
+            The created ScenarioTestMetric object.
+        """
+        response = self.connection.post(
+            AddScenarioTestMetric(
+                scenario_test_name=self.name,
+                eval_function_id=evaluation_criterion.eval_function_id,
+                threshold=evaluation_criterion.threshold,
+                threshold_comparison=evaluation_criterion.threshold_comparison,
+            ).dict(),
+            "validate/scenario_test_metric",
+        )
+        return ScenarioTestMetric(
+            scenario_test_id=response["scenario_test_id"],
+            eval_function_id=response["eval_function_id"],
+            threshold=evaluation_criterion.threshold,
+            threshold_comparison=evaluation_criterion.threshold_comparison,
+        )
+
+    def get_criteria(self) -> List[ScenarioTestMetric]:
+        """Retrieves all criteria of the :class:`ScenarioTest`. ::
+
+            import nucleus
+            client = nucleus.NucleusClient("YOUR_SCALE_API_KEY")
+            scenario_test = client.validate.list_scenario_tests()[0]
+
+            scenario_test.get_criteria()
+
+        Returns:
+            A list of ScenarioTestMetric objects.
+        """
+        response = self.connection.get(
+            f"validate/scenario_test/{self.id}/metrics",
+        )
+        return [
+            ScenarioTestMetric(**metric)
+            for metric in response["scenario_test_metrics"]
+        ]
+
+    def get_eval_history(self) -> List[ScenarioTestEvaluation]:
+        """Retrieves evaluation history for :class:`ScenarioTest`. ::
+
+            import nucleus
+            client = nucleus.NucleusClient("YOUR_SCALE_API_KEY")
+            scenario_test = client.validate.list_scenario_tests()[0]
+
+            scenario_test.get_eval_history()
+
+        Returns:
+            A list of :class:`ScenarioTestEvaluation` objects.
+        """
+        response = self.connection.get(
+            f"validate/scenario_test/{self.id}/eval_history",
+        )
+        eval_history = GetEvalHistory.parse_obj(response)
+        return [
+            ScenarioTestEvaluation(evaluation.id, self.connection)
+            for evaluation in eval_history.evaluations
+        ]
+
+    def get_items(self) -> List[DatasetItem]:
+        response = self.connection.get(
+            f"validate/scenario_test/{self.id}/items",
+        )
+        return [
+            DatasetItem.from_json(item) for item in response[DATASET_ITEMS_KEY]
+        ]

nucleus/validate/scenario_test_evaluation.py

@@ -0,0 +1,114 @@
+"""Data types for Scenario Test Evaluation results."""
+from dataclasses import InitVar, dataclass, field
+from enum import Enum
+from typing import List, Optional
+
+import requests
+
+from nucleus.connection import Connection
+from nucleus.constants import DATASET_ITEM_ID_KEY, MODEL_ID_KEY, STATUS_KEY
+from nucleus.validate.constants import (
+    EVAL_FUNCTION_ID_KEY,
+    EVALUATION_ID_KEY,
+    PASS_KEY,
+    RESULT_KEY,
+    SCENARIO_TEST_ID_KEY,
+)
+
+from .utils import try_convert_float
+
+SCENARIO_TEST_EVAL_KEY = "scenario_test_evaluation"
+ITEM_EVAL_KEY = "scenario_test_item_evaluations"
+
+
+class ScenarioTestEvaluationStatus(Enum):
+    """The Job status of scenario test evaluation."""
+
+    PENDING = "pending"
+    STARTED = "started"
+    COMPLETED = "completed"
+    ERRORED = "errored"
+
+
+@dataclass(frozen=True)
+class ScenarioTestItemEvaluation:
+    """Dataset item-level results of an evaluation of a scenario test.
+    Note that this class is immutable.
+
+    Attributes:
+        evaluation_id (str): The ID of the associated scenario test evaluation
+        scenario_test_id (str): The ID of the associated scenario test.
+        eval_function_id (str): The ID of the associated evaluation function.
+        dataset_item_id (str): The ID of the dataset item of this evaluation.
+        result (Optional[float]): The numerical result of the evaluation on this item.
+        passed (bool): Whether the result was sufficient to pass the test for this item.
+    """
+
+    evaluation_id: str
+    scenario_test_id: str
+    eval_function_id: str
+    dataset_item_id: str
+    result: Optional[float]
+    passed: bool
+
+
+@dataclass
+class ScenarioTestEvaluation:
+    """The results and attributes of an evaluation of a scenario test.
+
+    Attributes:
+        id (str): The ID of this scenario test evaluation.
+        scenario_test_id (str): The ID of the associated scenario test.
+        eval_function_id (str): The ID of the associated evaluation function.
+        model_id (str): THe ID of the associated model.
+        status (str): The status of the evaluation job.
+        result (Optional[float]): The float result of the evaluation.
+        passed (bool): Whether the scenario test was passed.
+        item_evals (List[ScenarioTestItemEvaluation]): The individual results for each dataset item.
+        connection (Connection): The connection to the Nucleus API.
+    """
+
+    # pylint: disable=too-many-instance-attributes
+
+    id: str
+    scenario_test_id: str = field(init=False)
+    eval_function_id: str = field(init=False)
+    model_id: str = field(init=False)
+    status: ScenarioTestEvaluationStatus = field(init=False)
+    result: Optional[float] = field(init=False)
+    passed: bool = field(init=False)
+    item_evals: List[ScenarioTestItemEvaluation] = field(init=False)
+    connection: InitVar[Connection]
+
+    def __post_init__(self, connection: Connection):
+        # TODO(gunnar): Having the function call /info on every construction is too slow. The original
+        #  endpoint should rather return the necessary human-readable information
+        response = connection.make_request(
+            {},
+            f"validate/eval/{self.id}/info",
+            requests_command=requests.get,
+        )
+        eval_response = response[SCENARIO_TEST_EVAL_KEY]
+        items_response = response[ITEM_EVAL_KEY]
+
+        self.scenario_test_id: str = eval_response[SCENARIO_TEST_ID_KEY]
+        self.eval_function_id: str = eval_response[EVAL_FUNCTION_ID_KEY]
+        self.model_id: str = eval_response[MODEL_ID_KEY]
+        self.status: ScenarioTestEvaluationStatus = (
+            ScenarioTestEvaluationStatus(eval_response[STATUS_KEY])
+        )
+        self.result: Optional[float] = try_convert_float(
+            eval_response[RESULT_KEY]
+        )
+        self.passed: bool = bool(eval_response[PASS_KEY])
+        self.item_evals: List[ScenarioTestItemEvaluation] = [
+            ScenarioTestItemEvaluation(
+                evaluation_id=res[EVALUATION_ID_KEY],
+                scenario_test_id=res[SCENARIO_TEST_ID_KEY],
+                eval_function_id=res[EVAL_FUNCTION_ID_KEY],
+                dataset_item_id=res[DATASET_ITEM_ID_KEY],
+                result=try_convert_float(res[RESULT_KEY]),
+                passed=bool(res[PASS_KEY]),
+            )
+            for res in items_response
+        ]

nucleus/validate/scenario_test_metric.py

@@ -0,0 +1,14 @@
+from nucleus.pydantic_base import ImmutableModel
+
+from .constants import ThresholdComparison
+
+
+class ScenarioTestMetric(ImmutableModel):
+    """A Scenario Test Metric is an evaluation function combined with a comparator and associated with a Scenario Test.
+    Scenario Test Metrics serve as the basis when evaluating a Model on a Scenario Test.
+    """
+
+    scenario_test_id: str
+    eval_function_id: str
+    threshold: float
+    threshold_comparison: ThresholdComparison

nucleus/validate/utils.py

@@ -0,0 +1,8 @@
+from typing import Optional
+
+
+def try_convert_float(float_str: str) -> Optional[float]:
+    try:
+        return float(float_str)
+    except ValueError:
+        return None