code-loader 1.0.64.dev1__py3-none-any.whl → 1.0.64.dev3__py3-none-any.whl

code_loader/default_metrics.py

@@ -0,0 +1,81 @@
+from enum import Enum
+from typing import List, Tuple
+import numpy as np
+
+from code_loader.contract.datasetclasses import ConfusionMatrixElement  # type: ignore
+from code_loader.contract.enums import ConfusionMatrixValue, MetricDirection  # type: ignore
+
+
+class Metric(Enum):
+    MeanSquaredError = 'MeanSquaredError'
+    MeanSquaredLogarithmicError = 'MeanSquaredLogarithmicError'
+    MeanAbsoluteError = 'MeanAbsoluteError'
+    MeanAbsolutePercentageError = 'MeanAbsolutePercentageError'
+    Accuracy = 'Accuracy'
+    ConfusionMatrixClassification = 'ConfusionMatrixClassification'
+
+
+def accuracy_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return np.mean((np.round(prediction).astype(np.bool_) == ground_truth.astype(np.bool_)), axis=1)
+
+
+def mean_squared_error_dimension_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return ((ground_truth - prediction) ** 2).mean(axis=1).astype(np.float32)
+
+
+def mean_absolute_error_dimension_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return np.abs(ground_truth - prediction).mean(axis=1).astype(np.float32)
+
+
+def mean_absolute_percentage_error_dimension_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return (np.abs(ground_truth - prediction) / np.abs(ground_truth)).mean(axis=1).astype(np.float32)
+
+
+def mean_squared_logarithmic_error_dimension_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return np.mean((np.log(1 + ground_truth) - np.log(1 + prediction)) ** 2, axis=1).astype(np.float32)
+
+
+def flatten_non_batch_dims(ground_truth: np.array, prediction: np.array) -> Tuple[np.array, np.array]:
+    batch_size = ground_truth.shape[0]
+    ground_truth = np.reshape(ground_truth, (batch_size, -1))
+    prediction = np.reshape(prediction, (batch_size, -1))
+    return ground_truth, prediction
+
+
+def confusion_matrix_classification_metric(ground_truth, prediction) -> List[List[ConfusionMatrixElement]]:
+    num_labels = prediction.shape[-1]
+    labels = [str(i) for i in range(num_labels)]
+    if len(labels) == 1:
+        labels = ['0', '1']
+        ground_truth = np.concatenate([1 - ground_truth, ground_truth], axis=1)
+        prediction = np.concatenate([1 - prediction, prediction], axis=1)
+
+    ret = []
+    for batch_i in range(ground_truth.shape[0]):
+        one_hot_vec = list(ground_truth[batch_i])
+        pred_vec = list(prediction[batch_i])
+        confusion_matrix_elements = []
+        for i, label in enumerate(labels):
+            expected_outcome = ConfusionMatrixValue.Positive if int(
+                one_hot_vec[i]) == 1 else ConfusionMatrixValue.Negative
+            cm_element = ConfusionMatrixElement(label, expected_outcome, float(pred_vec[i]))
+            confusion_matrix_elements.append(cm_element)
+        ret.append(confusion_matrix_elements)
+    return ret
+
+
+metrics_names_to_functions_and_direction = {
+    Metric.MeanSquaredError.name: (mean_squared_error_dimension_reduced, MetricDirection.Downward),
+    Metric.MeanSquaredLogarithmicError.name: (
+        mean_squared_logarithmic_error_dimension_reduced, MetricDirection.Downward),
+    Metric.MeanAbsoluteError.name: (mean_absolute_error_dimension_reduced, MetricDirection.Downward),
+    Metric.MeanAbsolutePercentageError.name: (
+        mean_absolute_percentage_error_dimension_reduced, MetricDirection.Downward),
+    Metric.Accuracy.name: (accuracy_reduced, MetricDirection.Upward),
+    Metric.ConfusionMatrixClassification.name: (confusion_matrix_classification_metric, None),
+}

code_loader/inner_leap_binder/leapbinder.py

@@ -14,6 +14,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
+from code_loader.default_metrics import metrics_names_to_functions_and_direction
 from code_loader.utils import to_numpy_return_wrapper, get_shape
 from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
     default_graph_visualizer, \
@@ -37,6 +38,7 @@ class LeapBinder:
         self._visualizer_names: List[str] = list()
         self._encoder_names: List[str] = list()
         self._extend_with_default_visualizers()
+        self._extend_with_default_metrics()
 
         self.batch_size_to_validate: Optional[int] = None
 
@@ -56,6 +58,10 @@ class LeapBinder:
         self.set_visualizer(function=default_text_mask_visualizer, name=DefaultVisualizer.TextMask.value,
                             visualizer_type=LeapDataType.TextMask)
 
+    def _extend_with_default_metrics(self) -> None:
+        for metric_name, (func, direction) in metrics_names_to_functions_and_direction.items():
+            self.add_custom_metric(func, metric_name, direction)
+
     def set_visualizer(self, function: VisualizerCallableInterface,
                        name: str,
                        visualizer_type: LeapDataType,

code_loader/leaploader.py

@@ -245,11 +245,12 @@ class LeapLoader(LeapLoaderBase):
         return self._visualizer_handler_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
 
     def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
-                               ) -> npt.NDArray[np.float32]:
+                               ) -> Optional[npt.NDArray[np.float32]]:
         heatmap_function = self._visualizer_handler_by_name()[visualizer_name].heatmap_function
         if heatmap_function is None:
             assert len(input_tensors_by_arg_name) == 1
-            return list(input_tensors_by_arg_name.values())[0]
+            return None
+
         return heatmap_function(**input_tensors_by_arg_name)
 
     def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:

code_loader/leaploaderbase.py

@@ -64,7 +64,7 @@ class LeapLoaderBase:
 
     @abstractmethod
     def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
-                               ) -> npt.NDArray[np.float32]:
+                               ) -> Optional[npt.NDArray[np.float32]]:
         pass
 
     @abstractmethod

{code_loader-1.0.64.dev1.dist-info → code_loader-1.0.64.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.64.dev1
+Version: 1.0.64.dev3
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.64.dev1.dist-info → code_loader-1.0.64.dev3.dist-info}/RECORD

@@ -7,6 +7,7 @@ code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=RSx9m_R3LawhK5o1nAcO3hfp2F9oJYtxZr_bpP3bTmw,4005
 code_loader/contract/visualizer_classes.py,sha256=iIa_O2rKvPTwN5ILCTZvRpsGYiiFABKdwQwfIXGigDo,11928
+code_loader/default_metrics.py,sha256=2kSaB71OrbQIXlcTMStYUebL8N-8bE9se3m-AXVdvCY,3936
 code_loader/experiment_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/experiment_api/api.py,sha256=a7wh6Hhe7IaVxu46eV2soSz-yxnmXG3ipU1BBtsEAaQ,2493
 code_loader/experiment_api/cli_config_utils.py,sha256=n6JMyNrquxql3KKxHhAP8jAzezlRT-PV2KWI95kKsm0,1140
@@ -18,14 +19,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=HDDs2ClP8xPbGaWUEYEHZgN7CTSliIpE5dly4Rawgrg,25352
+code_loader/inner_leap_binder/leapbinder.py,sha256=y_k7bRFYYmrZo4jCJrZ6mJykxc1slKDkODYCZ58OPs0,25691
 code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=I6ipji6QMN9qqFYxqQyNjtDtsnwpB-NahgKKpLPctMo,21067
-code_loader/leaploader.py,sha256=vssXiifNEGkfapjDtVd0Lpg3WKQ6qxYyCdoeGWvxiNc,22240
-code_loader/leaploaderbase.py,sha256=I1xwgGnDlvfyoOhlpKXXEuCqJUs0jqPzk3DTQaE6riQ,3080
+code_loader/leaploader.py,sha256=K__WKfqtKwEch40au177Po10EUX7gm0PJzcV6kpUMlo,22212
+code_loader/leaploaderbase.py,sha256=aHlqWDZRacIdBefeB9goYVnpApaNN2FT24uPIWKkCeQ,3090
 code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=VoqO9FN84yXyMjRjHjUTOt2GdTkJRMbHbXJ1cJkREkk,2230
-code_loader-1.0.64.dev1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.64.dev1.dist-info/METADATA,sha256=61TPcF5ReNA-Em_kAzM0NkW_ye_3_hNffl3mpaOrqUg,854
-code_loader-1.0.64.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.64.dev1.dist-info/RECORD,,
+code_loader-1.0.64.dev3.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.64.dev3.dist-info/METADATA,sha256=ICTjVXWwKwts78r-aY6VX0RzsYxfUpkOTaVeurAc8sk,854
+code_loader-1.0.64.dev3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.64.dev3.dist-info/RECORD,,