code-loader 1.0.174.dev2__tar.gz → 1.0.174.dev4__tar.gz

{code_loader-1.0.174.dev2 → code_loader-1.0.174.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.174.dev2
+Version: 1.0.174.dev4
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.174.dev2 → code_loader-1.0.174.dev4}/code_loader/inner_leap_binder/leapbinder.py

@@ -12,7 +12,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler
+    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler, InstanceMetricHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
@@ -335,6 +335,49 @@ class LeapBinder:
         metric_handler_data = MetricHandlerData(name, regular_arg_names, direction, compute_insights)
         self.setup_container.metrics.append(MetricHandler(metric_handler_data, function))
 
+    def add_custom_instance_metric(self,
+                          function: Union[CustomCallableInterfaceMultiArgs,
+                          CustomMultipleReturnCallableInterfaceMultiArgs,
+                          ConfusionMatrixCallableInterfaceMultiArgs],
+                          name: str,
+                          direction: Optional[
+                              Union[MetricDirection, Dict[str, MetricDirection]]] = MetricDirection.Downward,
+                          compute_insights: Optional[Union[bool, Dict[str, bool]]] = None) -> None:
+        """
+        Add a custom metric to the setup.
+
+        Args:
+        function (Union[CustomCallableInterfaceMultiArgs, CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]): The custom metric function.
+        name (str): The name of the custom metric.
+        direction (Optional[Union[MetricDirection, Dict[str, MetricDirection]]]): The direction of the metric, either
+        MetricDirection.Upward or MetricDirection.Downward, in case custom metric return a dictionary of metrics we can
+        supply a dictionary of directions correspondingly.
+            - MetricDirection.Upward: Indicates that higher values of the metric are better and should be maximized.
+            - MetricDirection.Downward: Indicates that lower values of the metric are better and should be minimized.
+        compute_insights (Union[bool, Dict[str, bool]]): Whether to compute insights or not. in case custom metric
+        return a dictionary of metrics we can supply a dictionary of values correspondingly
+
+
+
+        Example:
+            def custom_metric_function(y_true, y_pred):
+                return np.mean(np.abs(y_true - y_pred))
+
+            leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
+        """
+
+        regular_arg_names = inspect.getfullargspec(function)[0]
+        preprocess_response_arg_name = None
+        for arg_name, arg_type in inspect.getfullargspec(function).annotations.items():
+            if arg_type == SamplePreprocessResponse:
+                if preprocess_response_arg_name is not None:
+                    raise Exception("only one argument can be of type SamplePreprocessResponse")
+                preprocess_response_arg_name = arg_name
+                regular_arg_names.remove(arg_name)
+
+        metric_handler_data = MetricHandlerData(name, regular_arg_names, direction, compute_insights)
+        self.setup_container.instance_metrics.append(InstanceMetricHandler(metric_handler_data, function))
+
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1, prediction_index: Optional[int]=None) -> None:
         """
         Add prediction labels to the setup.

{code_loader-1.0.174.dev2 → code_loader-1.0.174.dev4}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1037,7 +1037,7 @@ def tensorleap_custom_instances_metric(name: str,
         except (TypeError, ValueError):
             pass
 
-        leap_binder.add_custom_metric(inner_without_validate, name, direction, compute_insights)
+        leap_binder.add_custom_instance_metric(inner_without_validate, name, direction, compute_insights)
 
         if connects_to is not None:
             arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names

{code_loader-1.0.174.dev2 → code_loader-1.0.174.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.174.dev2"
+version = "1.0.174.dev4"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"