optima-ml 0.3.4a3__tar.gz → 0.3.5.dev1__tar.gz

{optima_ml-0.3.4a3 → optima_ml-0.3.5.dev1}/OPTIMA/builtin/evaluation.py

@@ -9,8 +9,6 @@ import matplotlib.pyplot as plt
 import seaborn as sns
 from sklearn.metrics import roc_curve, auc
 
-import ray
-
 import OPTIMA.core.model
 import OPTIMA.core.evaluation
 
@@ -18,14 +16,14 @@ import OPTIMA.core.evaluation
 def evaluate(
     run_config,
     model_path,
-    inputs_split,
-    targets_split,
-    weights_split,
-    normalized_weights_split,
+    model_config,
+    dataset_split,
+    input_handler,
     fig_dir,
     native_metrics=None,
     weighted_native_metrics=None,
-    custom_FoMs=None,
+    custom_metrics=None,
+    composite_metrics=None,
     class_labels=None,
     cpus=1,
     results_dir=None,
@@ -34,7 +32,9 @@ def evaluate(
     return_unfilled=False,
     ratio=True,
 ):
-    """_summary_.
+    """Built-in evaluation for classification tasks.
+
+    This function assumes that the entire dataset and the model predictions can fit into memory.
 
     Parameters
     ----------
@@ -42,13 +42,11 @@ def evaluate(
         _description_
     model_path : _type_
         _description_
-    inputs_split : _type_
-        _description_
-    targets_split : _type_
+    model_config : _type_
         _description_
-    weights_split : _type_
+    dataset_split : _type_
         _description_
-    normalized_weights_split : _type_
+    input_handler : _type_
         _description_
     fig_dir : _type_
         _description_
@@ -56,8 +54,10 @@ def evaluate(
         _description_ (Default value = [])
     weighted_native_metrics : _type_
         _description_ (Default value = [])
-    custom_FoMs : _type_
+    custom_metrics : _type_
         _description_ (Default value = [])
+    composite_metrics : _type_
+        _description_ (Default value = None)
     class_labels : _type_
         _description_ (Default value = None)
     cpus : _type_
@@ -78,34 +78,62 @@ def evaluate(
     _type_
         _description_
     """
-    if custom_FoMs is None:
-        custom_FoMs = []
-    if weighted_native_metrics is None:
-        weighted_native_metrics = []
     if native_metrics is None:
         native_metrics = []
-
-    # fetch the inputs from the object store
-    if len(inputs_split) == 2:
+    if weighted_native_metrics is None:
+        weighted_native_metrics = []
+    if custom_metrics is None:
+        custom_metrics = []
+    if composite_metrics is None:
+        composite_metrics = []
+
+    # fetch the target labels and optional sample weights --> this saves everything into this worker's memory. If OOM
+    # error occurs, define an evaluate-function in the run-config.
+    if len(dataset_split) == 2:
         explicit_testing_dataset = False
-        inputs_train, inputs_val = ray.get(inputs_split)
-        targets_train, targets_val = ray.get(targets_split)
-        weights_train, weights_val = ray.get(weights_split)
-        normalized_weights_train, normalized_weights_val = ray.get(normalized_weights_split)
+        dataset_train, dataset_val = dataset_split
+        train_data = list(dataset_train.iter_batches(batch_size=dataset_train.count()))[0]
+        val_data = list(dataset_val.iter_batches(batch_size=dataset_val.count()))[0]
+        targets_train = train_data["Target"]
+        targets_val = val_data["Target"]
+        if "ScaledWeight" in dataset_train.columns():
+            weights_train = train_data["ScaledWeight"]
+            weights_val = val_data["ScaledWeight"]
+        elif "Weight" in dataset_train.columns():
+            weights_train = train_data["Weight"]
+            weights_val = val_data["Weight"]
+        else:
+            weights_train = np.ones((targets_train.shape[0],))
+            weights_val = np.ones((targets_val.shape[0],))
         print(
             "testing model using {} training and {} validation events".format(
-                inputs_train.shape[0], inputs_val.shape[0]
+                targets_train.shape[0], targets_val.shape[0]
             )
         )
     else:
         explicit_testing_dataset = True
-        inputs_train, inputs_val, inputs_test = ray.get(inputs_split)
-        targets_train, targets_val, targets_test = ray.get(targets_split)
-        weights_train, weights_val, weights_test = ray.get(weights_split)
-        normalized_weights_train, normalized_weights_val, normalized_weights_test = ray.get(normalized_weights_split)
+        dataset_train, dataset_val, dataset_test = dataset_split
+        train_data = list(dataset_train.iter_batches(batch_size=dataset_train.count()))[0]
+        val_data = list(dataset_val.iter_batches(batch_size=dataset_val.count()))[0]
+        test_data = list(dataset_test.iter_batches(batch_size=dataset_test.count()))[0]
+        targets_train = train_data["Target"]
+        targets_val = val_data["Target"]
+        targets_test = test_data["Target"]
+        if "ScaledWeight" in dataset_train.columns():
+            weights_train = train_data["ScaledWeight"]
+            weights_val = val_data["ScaledWeight"]
+            weights_test = test_data["ScaledWeight"]
+        elif "Weight" in dataset_train.columns():
+            weights_train = train_data["Weight"]
+            weights_val = val_data["Weight"]
+            weights_test = test_data["Weight"]
+        else:
+            weights_train = np.ones((targets_train.shape[0],))
+            weights_val = np.ones((targets_val.shape[0],))
+            weights_test = np.ones((targets_test.shape[0],))
         print(
             "testing model using {} training, {} validation and {} testing events".format(
-                inputs_train.shape[0], inputs_val.shape[0], inputs_test.shape[0]
+                targets_train.shape[0], targets_val.shape[0], targets_test.shape[0]
             )
         )
 
@@ -116,12 +144,19 @@ def evaluate(
         if not os.path.exists(results_dir):
             os.makedirs(results_dir, exist_ok=True)
 
-    # load the model and get the model predictions
-    model = OPTIMA.core.model.load_model(run_config, model_path, cpus)
-    pred_train = model.predict(inputs_train, verbose=0)
-    pred_val = model.predict(inputs_val, verbose=0)
+    # load the model, prepare the datasets (i.e. convert from ray datasets to ML native datasets) and get the model
+    # predictions
+    model = OPTIMA.core.model.load_model(run_config, model_config, input_handler, model_path, cpus)
+    native_datasets = model.prepare_datasets(
+        dataset_train,
+        dataset_val,
+        dataset_test if explicit_testing_dataset else None,
+    )
+    preds = model.predict(native_datasets, verbose=0)
     if explicit_testing_dataset:
-        pred_test = model.predict(inputs_test, verbose=0)
+        pred_train, pred_val, pred_test = preds
+    else:
+        pred_train, pred_val = preds
     num_outputs = pred_train.shape[1]
 
     # check if we have binary or multiclass classification
@@ -715,88 +750,130 @@ def evaluate(
             if explicit_testing_dataset:
                 results_string_args.append(auc_test_classes[i])
 
+    # keep track of the calculated metrics to evaluate composite metrics
+    metric_values_dict = {}
+
     # loss
     results_string += " Loss:\n"
-    train_loss = model.loss(
-        inputs=inputs_train, y_true=targets_train, sample_weight=normalized_weights_train, y_pred=pred_train
-    )
-    val_loss = model.loss(inputs=inputs_val, y_true=targets_val, sample_weight=normalized_weights_val, y_pred=pred_val)
+    losses = model.loss(native_datasets)
     if explicit_testing_dataset:
-        test_loss = model.loss(
-            inputs=inputs_test, y_true=targets_test, sample_weight=normalized_weights_test, y_pred=pred_test
-        )
+        train_loss, val_loss, test_loss = losses
+    else:
+        train_loss, val_loss = losses
     results_string += "\ttraining: {}\n".format("{:.3f}")
     results_string += "\tvalidation: {}\n".format("{:.3f}")
     if explicit_testing_dataset:
         results_string += "\ttesting: {}\n".format("{:.3f}")
-    results_string_args += [train_loss, val_loss, test_loss] if explicit_testing_dataset else [train_loss, val_loss]
+        metric_values_dict["loss"] = train_loss
+        metric_values_dict["val_loss"] = val_loss
+        metric_values_dict["test_loss"] = test_loss
+        results_string_args += [train_loss, val_loss, test_loss]
+    else:
+        metric_values_dict["loss"] = train_loss
+        metric_values_dict["val_loss"] = val_loss
+        results_string_args += [train_loss, val_loss]
 
     if native_metrics != []:
         # instantiate native metrics
         native_metrics = [(name, metric(**kwargs)) for name, (metric, kwargs) in native_metrics]
 
+        # calculate the metric values
+        metric_values = model.calc_native_metrics(
+            native_metrics=[metric for (_, metric) in native_metrics],
+            data=native_datasets,
+            weighted=False,
+        )
+        if explicit_testing_dataset:
+            metric_values_train, metric_values_val, metric_values_test = metric_values
+        else:
+            metric_values_train, metric_values_val = metric_values
+
         results_string += " Native metrics:\n"
-        for metric_name, metric in native_metrics:
-            metric_value_train = OPTIMA.core.evaluation.calc_native_metric(
-                run_config, metric, targets_train, pred_train
-            )
-            metric_value_val = OPTIMA.core.evaluation.calc_native_metric(run_config, metric, targets_val, pred_val)
+        for i, (metric_name, _) in enumerate(native_metrics):
+            metric_value_train = metric_values_train[i]
+            metric_value_val = metric_values_val[i]
             if explicit_testing_dataset:
-                metric_value_test = OPTIMA.core.evaluation.calc_native_metric(
-                    run_config, metric, targets_test, pred_test
-                )
+                metric_value_test = metric_values_test[i]
+
             results_string += "\t{} (training): {}\n".format(metric_name, "{:.3f}")
             results_string += "\t{} (validation): {}\n".format(metric_name, "{:.3f}")
             if explicit_testing_dataset:
                 results_string += "\t{} (testing): {}\n".format(metric_name, "{:.3f}")
-            results_string_args += (
-                [metric_value_train, metric_value_val, metric_value_test]
-                if explicit_testing_dataset
-                else [metric_value_train, metric_value_val]
-            )
+                metric_values_dict[f"{metric_name}"] = metric_value_train
+                metric_values_dict[f"val_{metric_name}"] = metric_value_val
+                metric_values_dict[f"test_{metric_name}"] = metric_value_test
+                results_string_args += [metric_value_train, metric_value_val, metric_value_test]
+            else:
+                metric_values_dict[f"{metric_name}"] = metric_value_train
+                metric_values_dict[f"val_{metric_name}"] = metric_value_val
+                results_string_args += [metric_value_train, metric_value_val]
 
     if weighted_native_metrics != []:
         # instantiate weighted native metrics
         weighted_native_metrics = [(name, metric(**kwargs)) for name, (metric, kwargs) in weighted_native_metrics]
 
+        # calculate the metric values
+        metric_values = model.calc_native_metrics(
+            native_metrics=[metric for (_, metric) in weighted_native_metrics],
+            data=native_datasets,
+            weighted=True,
+        )
+        if explicit_testing_dataset:
+            metric_values_train, metric_values_val, metric_values_test = metric_values
+        else:
+            metric_values_train, metric_values_val = metric_values
+
         results_string += " Weighted native metrics:\n"
-        for metric_name, metric in weighted_native_metrics:
-            metric_value_train = OPTIMA.core.evaluation.calc_native_metric(
-                run_config, metric, targets_train, pred_train, sample_weight=normalized_weights_train
-            )
-            metric_value_val = OPTIMA.core.evaluation.calc_native_metric(
-                run_config, metric, targets_val, pred_val, sample_weight=normalized_weights_val
-            )
+        for i, (metric_name, _) in enumerate(weighted_native_metrics):
+            metric_value_train = metric_values_train[i]
+            metric_value_val = metric_values_val[i]
             if explicit_testing_dataset:
-                metric_value_test = OPTIMA.core.evaluation.calc_native_metric(
-                    run_config, metric, targets_test, pred_test, sample_weight=normalized_weights_test
-                )
+                metric_value_test = metric_values_test[i]
+
             results_string += "\t{} (training): {}\n".format(metric_name, "{:.3f}")
             results_string += "\t{} (validation): {}\n".format(metric_name, "{:.3f}")
             if explicit_testing_dataset:
                 results_string += "\t{} (testing): {}\n".format(metric_name, "{:.3f}")
-            results_string_args += (
-                [metric_value_train, metric_value_val, metric_value_test]
-                if explicit_testing_dataset
-                else [metric_value_train, metric_value_val]
-            )
+                metric_values_dict[f"{metric_name}"] = metric_value_train
+                metric_values_dict[f"val_{metric_name}"] = metric_value_val
+                metric_values_dict[f"test_{metric_name}"] = metric_value_test
+                results_string_args += [metric_value_train, metric_value_val, metric_value_test]
+            else:
+                metric_values_dict[f"{metric_name}"] = metric_value_train
+                metric_values_dict[f"val_{metric_name}"] = metric_value_val
+                results_string_args += [metric_value_train, metric_value_val]
 
-    if custom_FoMs != []:
+    if custom_metrics != []:
         results_string += " Custom metrics:\n"
-        for FoM_name, FoM_func in custom_FoMs:
-            FoM_value_train = FoM_func(targets_train, pred_train, sample_weight=normalized_weights_train)
-            FoM_value_val = FoM_func(targets_val, pred_val, sample_weight=normalized_weights_val)
-            if explicit_testing_dataset:
-                FoM_value_test = FoM_func(targets_test, pred_test, sample_weight=normalized_weights_test)
-            results_string += "\t{} (training): {}\n".format(FoM_name, "{:.3f}")
-            results_string += "\t{} (validation): {}\n".format(FoM_name, "{:.3f}")
+        custom_metric_values = model.calc_custom_metrics(
+            custom_metrics=[metric for (_, metric) in custom_metrics],
+            data=native_datasets,
+            skip_test_dataset=False,
+        )
+        for i, (metric_name, _) in enumerate(custom_metrics):
+            results_string += "\t{} (training): {}\n".format(metric_name, "{:.3f}")
+            results_string += "\t{} (validation): {}\n".format(metric_name, "{:.3f}")
             if explicit_testing_dataset:
-                results_string += "\t{} (testing): {}\n".format(FoM_name, "{:.3f}")
-            results_string_args += (
-                [FoM_value_train, FoM_value_val, FoM_value_test]
-                if explicit_testing_dataset
-                else [FoM_value_train, FoM_value_val]
-            )
+                results_string += "\t{} (testing): {}\n".format(metric_name, "{:.3f}")
+                metric_values_dict[f"train_{metric_name}"] = custom_metric_values[0][i]
+                metric_values_dict[f"val_{metric_name}"] = custom_metric_values[1][i]
+                metric_values_dict[f"test_{metric_name}"] = custom_metric_values[2][i]
+                results_string_args += [
+                    custom_metric_values[0][i],
+                    custom_metric_values[1][i],
+                    custom_metric_values[2][i],
+                ]
+            else:
+                metric_values_dict[f"train_{metric_name}"] = custom_metric_values[0][i]
+                metric_values_dict[f"val_{metric_name}"] = custom_metric_values[1][i]
+                results_string_args += [custom_metric_values[0][i], custom_metric_values[1][i]]
+
+    if composite_metrics != []:
+        results_string += " Composite metrics:\n"
+        for metric_name, dep_metric_names, metric in composite_metrics:
+            dep_metric_values = (metric_values_dict[dep_metric_name] for dep_metric_name in dep_metric_names)
+            results_string += "\t{}: {}\n".format(metric_name, "{:.3f}")
+            results_string_args.append(metric(*dep_metric_values))
 
     if print_results:
         print(results_string.format(*results_string_args))

{optima_ml-0.3.4a3 → optima_ml-0.3.5.dev1}/OPTIMA/builtin/figures_of_merit.py

@@ -95,24 +95,28 @@ class FigureOfMerit:
         self.func = func
         self.kwargs = kwargs
 
-    def __call__(self, y_true: np.ndarray, y_pred: np.ndarray, sample_weight: Optional[np.ndarray] = None) -> Any:
+    def __call__(self, inputs: tuple[np.ndarray], y_pred: np.ndarray) -> Any:
         """Gives arrays of target labels, predictions, sample weight and possible kwargs to self.func and returns the result.
 
         Parameters
         ----------
-        y_true : np.ndarray
-            Array of target labels.
+        inputs : tuple[np.ndarray]
+            A tuple of numpy arrays of length 2 or 3. The first two entries are assumed to be the input features and
+            the target labels. The third, optional entry is assumed to be the sample weights.
         y_pred : np.ndarray
             Array of predictions corresponding to the target labels.
-        sample_weight : Optional[np.ndarray]
-            Array of sample weights for each entry in y_true and y_pred. (Default value = None)
 
         Returns
         -------
         Any
             Return value of the figure of merit calculated by self.func.
         """
-        return self.func(y_true, y_pred, sample_weight=sample_weight, **self.kwargs)
+        assert isinstance(inputs, tuple) or isinstance(inputs, list)
+        assert len(inputs) in [
+            2,
+            3,
+        ], f"An input-tuple or -list of length {len(inputs)} was provided, which is not supported."
+        return self.func(inputs[1], y_pred, sample_weight=inputs[2] if len(inputs) == 3 else None, **self.kwargs)
 
 
 class HistFigureOfMerit(FigureOfMerit):
@@ -132,7 +136,7 @@ class HistFigureOfMerit(FigureOfMerit):
         min_events_per_bin: Union[float, int] = 10.0,
         min_s_per_bin: Union[float, int] = 0,
         min_b_per_bin: Union[float, int] = 1.0,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> None:
         """Constructs a HistFigureOfMerit object.
 
@@ -169,24 +173,28 @@ class HistFigureOfMerit(FigureOfMerit):
         self.min_s_per_bin = min_s_per_bin
         self.min_b_per_bin = min_b_per_bin
 
-    def __call__(self, y_true: np.ndarray, y_pred: np.ndarray, sample_weight: Optional[np.ndarray] = None) -> Any:
+    def __call__(self, inputs: tuple[np.ndarray], y_pred: np.ndarray) -> Any:
         """Calls the preprocess function to build the histograms, gives them to self.func and returns the result.
 
         Parameters
         ----------
-        y_true : np.ndarray
-            Array of target labels.
+        inputs : tuple[np.ndarray]
+            A tuple of numpy arrays of length 2 or 3. The first two entries are assumed to be the input features and
+            the target labels. The third, optional entry is assumed to be the sample weights.
         y_pred : np.ndarray
             Array of predictions corresponding to the target labels.
-        sample_weight : Optional[np.ndarray]
-            Array of sample weights for each entry in y_true and y_pred. (Default value = None)
 
         Returns
         -------
         Any
             Value of the figure of merit calculated by self.func.
         """
-        s, b, bins = self.preprocess(y_true, y_pred, sample_weight=sample_weight)
+        assert isinstance(inputs, tuple) or isinstance(inputs, list)
+        assert len(inputs) in [
+            2,
+            3,
+        ], f"An input-tuple or -list of length {len(inputs)} was provided, which is not supported."
+        s, b, bins = self.preprocess(inputs[1], y_pred, sample_weight=inputs[2] if len(inputs) == 3 else None)
         return self.func(s, b, bins, **self.kwargs)
 
     def preprocess(
@@ -295,7 +303,7 @@ class NormHistFigureOfMerit(FigureOfMerit):
         min_events_per_bin: Union[float, int] = 0.0,
         min_s_per_bin: Union[float, int] = 0,
         min_b_per_bin: Union[float, int] = 0.0,
-        **kwargs: Any
+        **kwargs: Any,
     ) -> None:
         """_summary_.
 
@@ -331,24 +339,28 @@ class NormHistFigureOfMerit(FigureOfMerit):
         self.min_s_per_bin = min_s_per_bin
         self.min_b_per_bin = min_b_per_bin
 
-    def __call__(self, y_true: np.ndarray, y_pred: np.ndarray, sample_weight: Optional[np.ndarray] = None) -> Any:
+    def __call__(self, inputs: tuple[np.ndarray], y_pred: np.ndarray) -> Any:
         """Calls the preprocess function to build the histograms, gives them to self.func and returns the result.
 
         Parameters
         ----------
-        y_true : np.ndarray
-            Array of target labels.
+        inputs : tuple[np.ndarray]
+            A tuple of numpy arrays of length 2 or 3. The first two entries are assumed to be the input features and
+            the target labels. The third, optional entry is assumed to be the sample weights.
         y_pred : np.ndarray
             Array of predictions corresponding to the target labels.
-        sample_weight : Optional[np.ndarray]
-            Array of sample weights for each entry in y_true and y_pred. (Default value = None)
 
         Returns
         -------
         Any
             Value of the figure of merit calculated by self.func.
         """
-        s, b, bins = self.preprocess(y_true, y_pred, sample_weight=sample_weight)
+        assert isinstance(inputs, tuple) or isinstance(inputs, list)
+        assert len(inputs) in [
+            2,
+            3,
+        ], f"An input-tuple or -list of length {len(inputs)} was provided, which is not supported."
+        s, b, bins = self.preprocess(inputs[1], y_pred, sample_weight=inputs[2] if len(inputs) == 3 else None)
         return self.func(s, b, bins, **self.kwargs)
 
     def preprocess(