code-loader 1.0.174.dev1__tar.gz → 1.0.174.dev3__tar.gz

{code_loader-1.0.174.dev1 → code_loader-1.0.174.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.174.dev1
+Version: 1.0.174.dev3
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.174.dev1 → code_loader-1.0.174.dev3}/code_loader/contract/datasetclasses.py

@@ -264,7 +264,7 @@ class DatasetIntegrationSetup:
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)
     custom_loss_handlers: List[CustomLossHandler] = field(default_factory=list)
     metrics: List[MetricHandler] = field(default_factory=list)
-    instance_metrics: List[MetricHandler] = field(default_factory=list)
+    instance_metrics: List[InstanceMetricHandler] = field(default_factory=list)
     custom_layers: Dict[str, CustomLayerHandler] = field(default_factory=dict)
     custom_latent_space: Optional[CustomLatentSpaceHandler] = None
 

{code_loader-1.0.174.dev1 → code_loader-1.0.174.dev3}/code_loader/inner_leap_binder/leapbinder.py

@@ -12,7 +12,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler
+    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler, InstanceMetricHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload, LeapAnalysisConfiguration
@@ -335,6 +335,49 @@ class LeapBinder:
         metric_handler_data = MetricHandlerData(name, regular_arg_names, direction, compute_insights)
         self.setup_container.metrics.append(MetricHandler(metric_handler_data, function))
 
+    def add_custom_instance_metric(self,
+                          function: Union[CustomCallableInterfaceMultiArgs,
+                          CustomMultipleReturnCallableInterfaceMultiArgs,
+                          ConfusionMatrixCallableInterfaceMultiArgs],
+                          name: str,
+                          direction: Optional[
+                              Union[MetricDirection, Dict[str, MetricDirection]]] = MetricDirection.Downward,
+                          compute_insights: Optional[Union[bool, Dict[str, bool]]] = None) -> None:
+        """
+        Add a custom metric to the setup.
+
+        Args:
+        function (Union[CustomCallableInterfaceMultiArgs, CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]): The custom metric function.
+        name (str): The name of the custom metric.
+        direction (Optional[Union[MetricDirection, Dict[str, MetricDirection]]]): The direction of the metric, either
+        MetricDirection.Upward or MetricDirection.Downward, in case custom metric return a dictionary of metrics we can
+        supply a dictionary of directions correspondingly.
+            - MetricDirection.Upward: Indicates that higher values of the metric are better and should be maximized.
+            - MetricDirection.Downward: Indicates that lower values of the metric are better and should be minimized.
+        compute_insights (Union[bool, Dict[str, bool]]): Whether to compute insights or not. in case custom metric
+        return a dictionary of metrics we can supply a dictionary of values correspondingly
+
+
+
+        Example:
+            def custom_metric_function(y_true, y_pred):
+                return np.mean(np.abs(y_true - y_pred))
+
+            leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
+        """
+
+        regular_arg_names = inspect.getfullargspec(function)[0]
+        preprocess_response_arg_name = None
+        for arg_name, arg_type in inspect.getfullargspec(function).annotations.items():
+            if arg_type == SamplePreprocessResponse:
+                if preprocess_response_arg_name is not None:
+                    raise Exception("only one argument can be of type SamplePreprocessResponse")
+                preprocess_response_arg_name = arg_name
+                regular_arg_names.remove(arg_name)
+
+        metric_handler_data = MetricHandlerData(name, regular_arg_names, direction, compute_insights)
+        self.setup_container.instance_metrics.append(InstanceMetricHandler(metric_handler_data, function))
+
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1, prediction_index: Optional[int]=None) -> None:
         """
         Add prediction labels to the setup.

{code_loader-1.0.174.dev1 → code_loader-1.0.174.dev3}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -879,8 +879,7 @@ def tensorleap_custom_metric(name: str,
 
 def tensorleap_custom_instances_metric(name: str,
                              direction: Union[MetricDirection, Dict[str, MetricDirection]] = _UNSET,
-                             compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
-                             connects_to=None):
+                             compute_insights: Optional[Union[bool, Dict[str, bool]]] = None):
     name_to_unique_name = defaultdict(set)
 
     def decorating_function(
@@ -943,21 +942,21 @@ def tensorleap_custom_instances_metric(name: str,
                                 err_message +
                                 f"Invalid type for compute_insights['{k}']: expected bool, got type {type(v).__name__}."
                             )
-            if connects_to is not None:
-                valid_types = (str, list, tuple, set)
-                if not isinstance(connects_to, valid_types):
-                    raise TypeError(
-                        err_message +
-                        f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
-                    )
-                if isinstance(connects_to, (list, tuple, set)):
-                    invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
-                    if invalid_elems:
-                        raise TypeError(
-                            err_message +
-                            f"All elements in `connects_to` must be strings, "
-                            f"but found element types: {invalid_elems}."
-                        )
+            # if connects_to is not None:
+            #     valid_types = (str, list, tuple, set)
+            #     if not isinstance(connects_to, valid_types):
+            #         raise TypeError(
+            #             err_message +
+            #             f"`connects_to` must be one of {valid_types}, got type {type(connects_to).__name__}."
+            #         )
+            #     if isinstance(connects_to, (list, tuple, set)):
+            #         invalid_elems = [f"{type(e).__name__}" for e in connects_to if not isinstance(e, str)]
+            #         if invalid_elems:
+            #             raise TypeError(
+            #                 err_message +
+            #                 f"All elements in `connects_to` must be strings, "
+            #                 f"but found element types: {invalid_elems}."
+            #             )
 
         _validate_decorators_signature()
 
@@ -1037,11 +1036,11 @@ def tensorleap_custom_instances_metric(name: str,
         except (TypeError, ValueError):
             pass
 
-        leap_binder.add_custom_metric(inner_without_validate, name, direction, compute_insights)
+        leap_binder.add_custom_instance_metric(inner_without_validate, name, direction, compute_insights)
 
-        if connects_to is not None:
-            arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
-            _add_mapping_connections(connects_to, arg_names, NodeMappingType.Metric, name)
+        # if connects_to is not None:
+        #     arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
+        #     _add_mapping_connections(connects_to, arg_names, NodeMappingType.Metric, name)
 
         def inner(*args, **kwargs):
             if not _call_from_tl_platform:

{code_loader-1.0.174.dev1 → code_loader-1.0.174.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.174.dev1"
+version = "1.0.174.dev3"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"