code-loader 1.0.71__tar.gz → 1.0.72.dev1__tar.gz

{code_loader-1.0.71 → code_loader-1.0.72.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.71
+Version: 1.0.72.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.71 → code_loader-1.0.72.dev1}/code_loader/contract/datasetclasses.py

@@ -132,7 +132,7 @@ class CustomLossHandler:
 class MetricHandlerData:
     name: str
     arg_names: List[str]
-    direction: Optional[MetricDirection] = MetricDirection.Downward
+    direction: Union[None, MetricDirection, Dict[str, MetricDirection]] = MetricDirection.Downward
 
 
 @dataclass
@@ -146,6 +146,12 @@ class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
 
 
+@dataclass
+class SamplePreprocessResponse:
+    sample_ids: np.array
+    preprocess_response: PreprocessResponse
+
+
 @dataclass
 class VisualizerHandlerData:
     name: str
@@ -171,6 +177,7 @@ class InputHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
     channel_dim: Optional[int] = -1
 
+
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None

{code_loader-1.0.71 → code_loader-1.0.72.dev1}/code_loader/contract/visualizer_classes.py

@@ -170,12 +170,13 @@ class LeapHorizontalBar:
     Example:
         body_data = np.random.rand(5).astype(np.float32)
         labels = ['Class A', 'Class B', 'Class C', 'Class D', 'Class E']
-        leap_horizontal_bar = LeapHorizontalBar(body=body_data, labels=labels)
+        gt_data = np.array([0.1, 0.2, 0.3, 0.4, 0.5]).astype(np.float32)
+        leap_horizontal_bar = LeapHorizontalBar(body=body_data, labels=labels, gt=gt_data)
     """
     body: npt.NDArray[np.float32]
     labels: List[str]
-    type: LeapDataType = LeapDataType.HorizontalBar
     gt: Optional[npt.NDArray[np.float32]] = None
+    type: LeapDataType = LeapDataType.HorizontalBar
 
 
     def __post_init__(self) -> None:

{code_loader-1.0.71 → code_loader-1.0.72.dev1}/code_loader/inner_leap_binder/leapbinder.py

@@ -10,7 +10,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
-    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData
+    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
@@ -33,6 +33,7 @@ class LeapBinder:
     setup_container (DatasetIntegrationSetup): Container to hold setup configurations.
     cache_container (Dict[str, Any]): Cache container to store intermediate data.
     """
+
     def __init__(self) -> None:
         self.setup_container = DatasetIntegrationSetup()
         self.cache_container: Dict[str, Any] = {"word_to_index": {}}
@@ -238,26 +239,41 @@ class LeapBinder:
 
             leap_binder.add_custom_loss(custom_loss_function, name='custom_loss')
         """
-        arg_names = inspect.getfullargspec(function)[0]
-        self.setup_container.custom_loss_handlers.append(CustomLossHandler(CustomLossHandlerData(name, arg_names), function))
+
+        regular_arg_names = []
+        preprocess_response_arg_name = None
+        for arg_name, arg_type in inspect.getfullargspec(function).annotations.items():
+            if arg_type == SamplePreprocessResponse:
+                if preprocess_response_arg_name is not None:
+                    raise Exception("only one argument can be of type SamplePreprocessResponse")
+                preprocess_response_arg_name = arg_name
+            else:
+                regular_arg_names.append(arg_name)
+
+        self.setup_container.custom_loss_handlers.append(
+            CustomLossHandler(CustomLossHandlerData(name, regular_arg_names), function))
 
     def add_custom_metric(self,
                           function: Union[CustomCallableInterfaceMultiArgs,
                           CustomMultipleReturnCallableInterfaceMultiArgs,
                           ConfusionMatrixCallableInterfaceMultiArgs],
                           name: str,
-                          direction: Optional[MetricDirection] = MetricDirection.Downward) -> None:
+                          direction: Optional[
+                              Union[MetricDirection, Dict[str, MetricDirection]]] = MetricDirection.Downward) -> None:
         """
         Add a custom metric to the setup.
 
         Args:
         function (Union[CustomCallableInterfaceMultiArgs, CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]): The custom metric function.
         name (str): The name of the custom metric.
-        direction (Optional[MetricDirection]): The direction of the metric, either MetricDirection.Upward or MetricDirection.Downward.
+        direction (Optional[Union[MetricDirection, Dict[str, MetricDirection]]]): The direction of the metric, either
+        MetricDirection.Upward or MetricDirection.Downward, in case custom metric return a dictionary of metrics we can
+        supply a dictionary of directions correspondingly
             - MetricDirection.Upward: Indicates that higher values of the metric are better and should be maximized.
             - MetricDirection.Downward: Indicates that lower values of the metric are better and should be minimized.
 
 
+
         Example:
             def custom_metric_function(y_true, y_pred):
                 return np.mean(np.abs(y_true - y_pred))
@@ -377,7 +393,8 @@ class LeapBinder:
             custom_layer.kernel_index = kernel_index
 
         if use_custom_latent_space and not hasattr(custom_layer, custom_latent_space_attribute):
-            raise Exception(f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
+            raise Exception(
+                f"{custom_latent_space_attribute} function has not been set for custom layer: {custom_layer.__name__}")
 
         init_args = inspect.getfullargspec(custom_layer.__init__)[0][1:]
         call_args = inspect.getfullargspec(custom_layer.call)[0][1:]
@@ -490,7 +507,3 @@ class LeapBinder:
 
     def set_batch_size_to_validate(self, batch_size: int) -> None:
         self.batch_size_to_validate = batch_size
-
-
-
-

{code_loader-1.0.71 → code_loader-1.0.72.dev1}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1,6 +1,6 @@
 # mypy: ignore-errors
 
-from typing import Optional, Union, Callable, List
+from typing import Optional, Union, Callable, List, Dict
 
 import numpy as np
 import numpy.typing as npt
@@ -15,12 +15,11 @@ from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, Le
     LeapHorizontalBar, LeapImageWithBBox, LeapImageWithHeatmap
 
 
-def tensorleap_custom_metric(name: str, direction: Optional[MetricDirection] = MetricDirection.Downward):
-    def decorating_function(
-            user_function: Union[CustomCallableInterfaceMultiArgs,
-            CustomMultipleReturnCallableInterfaceMultiArgs,
-            ConfusionMatrixCallableInterfaceMultiArgs]
-    ):
+def tensorleap_custom_metric(name: str,
+                             direction: Union[MetricDirection, Dict[str, MetricDirection]] = MetricDirection.Downward):
+    def decorating_function(user_function: Union[CustomCallableInterfaceMultiArgs,
+    CustomMultipleReturnCallableInterfaceMultiArgs,
+    ConfusionMatrixCallableInterfaceMultiArgs]):
         for metric_handler in leap_binder.setup_container.metrics:
             if metric_handler.metric_handler_data.name == name:
                 raise Exception(f'Metric with name {name} already exists. '
@@ -356,15 +355,15 @@ def tensorleap_custom_loss(name: str):
                                                                f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
                 else:
                     assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
-                                                        f'Argument #{i} should be a numpy array. Got {type(arg)}.')
+                                                          f'Argument #{i} should be a numpy array. Got {type(arg)}.')
             for _arg_name, arg in kwargs.items():
                 if isinstance(arg, list):
                     for y, elem in enumerate(arg):
-                        assert isinstance(elem,valid_types), (f'tensorleap_custom_loss validation failed: '
-                                                             f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
+                        assert isinstance(elem, valid_types), (f'tensorleap_custom_loss validation failed: '
+                                                               f'Element #{y} of list should be a numpy array. Got {type(elem)}.')
                 else:
                     assert isinstance(arg, valid_types), (f'tensorleap_custom_loss validation failed: '
-                                                        f'Argument #{_arg_name} should be a numpy array. Got {type(arg)}.')
+                                                          f'Argument #{_arg_name} should be a numpy array. Got {type(arg)}.')
 
         def _validate_result(result):
             assert isinstance(result, valid_types), \

{code_loader-1.0.71 → code_loader-1.0.72.dev1}/code_loader/leaploader.py

@@ -14,7 +14,7 @@ import numpy.typing as npt
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
-    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap
+    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -233,9 +233,19 @@ class LeapLoader(LeapLoaderBase):
         self._preprocess_result()
         return self._metric_handler_by_name()[metric_name].function(**input_tensors_by_arg_name)
 
-    def run_custom_loss(self, custom_loss_name: str,
+    def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
-        return self._custom_loss_handler_by_name()[custom_loss_name].function(**input_tensors_by_arg_name)
+
+        custom_loss_handler = self._custom_loss_handler_by_name()[custom_loss_name]
+        preprocess_response_arg_name = None
+        for arg_name, arg_type in inspect.getfullargspec(custom_loss_handler.function).annotations.items():
+            if arg_type == SamplePreprocessResponse:
+                preprocess_response_arg_name = arg_name
+                break
+
+        if preprocess_response_arg_name is not None:
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(sample_ids, self._preprocess_result()[state])
+        return custom_loss_handler.function(**input_tensors_by_arg_name)
 
     def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
         # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
@@ -320,7 +330,8 @@ class LeapLoader(LeapLoaderBase):
             for visualizer_handler in setup.visualizers]
 
         custom_losses = [CustomLossInstance(custom_loss.custom_loss_handler_data.name,
-                                            custom_loss.custom_loss_handler_data.arg_names)
+                                            custom_loss.custom_loss_handler_data.arg_names,
+                                            custom_loss.custom_loss_handler_data.preprocess_response_arg_name)
                          for custom_loss in setup.custom_loss_handlers]
 
         prediction_types = []

{code_loader-1.0.71 → code_loader-1.0.72.dev1}/code_loader/leaploaderbase.py

@@ -20,6 +20,9 @@ class LeapLoaderBase:
         self.code_entry_name = code_entry_name
         self.code_path = code_path
 
+        self.current_working_sample_ids: Optional[np.array] = None
+        self.current_working_state: Optional[DataStateEnum] = None
+
     @abstractmethod
     def metric_by_name(self) -> Dict[str, MetricHandlerData]:
         pass
@@ -58,7 +61,7 @@ class LeapLoaderBase:
         pass
 
     @abstractmethod
-    def run_custom_loss(self, custom_loss_name: str,
+    def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
         pass
 

{code_loader-1.0.71 → code_loader-1.0.72.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.71"
+version = "1.0.72.dev1"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"