code-loader 1.0.72__py3-none-any.whl → 1.0.72.dev1__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -133,7 +133,6 @@ class MetricHandlerData:
     name: str
     arg_names: List[str]
     direction: Union[None, MetricDirection, Dict[str, MetricDirection]] = MetricDirection.Downward
-    compute_insights: Union[bool, Dict[str, bool]] = True
 
 
 @dataclass
@@ -147,6 +146,12 @@ class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
 
 
+@dataclass
+class SamplePreprocessResponse:
+    sample_ids: np.array
+    preprocess_response: PreprocessResponse
+
+
 @dataclass
 class VisualizerHandlerData:
     name: str

code_loader/inner_leap_binder/leapbinder.py

@@ -10,7 +10,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
-    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData
+    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
@@ -239,9 +239,19 @@ class LeapBinder:
 
             leap_binder.add_custom_loss(custom_loss_function, name='custom_loss')
         """
-        arg_names = inspect.getfullargspec(function)[0]
+
+        regular_arg_names = []
+        preprocess_response_arg_name = None
+        for arg_name, arg_type in inspect.getfullargspec(function).annotations.items():
+            if arg_type == SamplePreprocessResponse:
+                if preprocess_response_arg_name is not None:
+                    raise Exception("only one argument can be of type SamplePreprocessResponse")
+                preprocess_response_arg_name = arg_name
+            else:
+                regular_arg_names.append(arg_name)
+
         self.setup_container.custom_loss_handlers.append(
-            CustomLossHandler(CustomLossHandlerData(name, arg_names), function))
+            CustomLossHandler(CustomLossHandlerData(name, regular_arg_names), function))
 
     def add_custom_metric(self,
                           function: Union[CustomCallableInterfaceMultiArgs,
@@ -249,8 +259,7 @@ class LeapBinder:
                           ConfusionMatrixCallableInterfaceMultiArgs],
                           name: str,
                           direction: Optional[
-                              Union[MetricDirection, Dict[str, MetricDirection]]] = MetricDirection.Downward,
-                          compute_insights: Union[bool, Dict[str, bool]] = True) -> None:
+                              Union[MetricDirection, Dict[str, MetricDirection]]] = MetricDirection.Downward) -> None:
         """
         Add a custom metric to the setup.
 
@@ -259,11 +268,9 @@ class LeapBinder:
         name (str): The name of the custom metric.
         direction (Optional[Union[MetricDirection, Dict[str, MetricDirection]]]): The direction of the metric, either
         MetricDirection.Upward or MetricDirection.Downward, in case custom metric return a dictionary of metrics we can
-        supply a dictionary of directions correspondingly.
+        supply a dictionary of directions correspondingly
             - MetricDirection.Upward: Indicates that higher values of the metric are better and should be maximized.
             - MetricDirection.Downward: Indicates that lower values of the metric are better and should be minimized.
-        compute_insights (Union[bool, Dict[str, bool]]): Whether to compute insights or not. in case custom metric
-        return a dictionary of metrics we can supply a dictionary of values correspondingly
 
 
 
@@ -274,8 +281,7 @@ class LeapBinder:
             leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
         """
         arg_names = inspect.getfullargspec(function)[0]
-        metric_handler_data = MetricHandlerData(name, arg_names, direction, compute_insights)
-        self.setup_container.metrics.append(MetricHandler(metric_handler_data, function))
+        self.setup_container.metrics.append(MetricHandler(MetricHandlerData(name, arg_names, direction), function))
 
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1) -> None:
         """

code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -16,8 +16,7 @@ from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, Le
 
 
 def tensorleap_custom_metric(name: str,
-                             direction: Union[MetricDirection, Dict[str, MetricDirection]] = MetricDirection.Downward,
-                             compute_insights: Union[bool, Dict[str, bool]] = True):
+                             direction: Union[MetricDirection, Dict[str, MetricDirection]] = MetricDirection.Downward):
     def decorating_function(user_function: Union[CustomCallableInterfaceMultiArgs,
     CustomMultipleReturnCallableInterfaceMultiArgs,
     ConfusionMatrixCallableInterfaceMultiArgs]):
@@ -26,7 +25,7 @@ def tensorleap_custom_metric(name: str,
                 raise Exception(f'Metric with name {name} already exists. '
                                 f'Please choose another')
 
-        leap_binder.add_custom_metric(user_function, name, direction, compute_insights)
+        leap_binder.add_custom_metric(user_function, name, direction)
 
         def _validate_input_args(*args, **kwargs) -> None:
             for i, arg in enumerate(args):
@@ -76,19 +75,6 @@ def tensorleap_custom_metric(name: str,
                         (f'tensorleap_custom_metric validation failed: '
                          f'Keys in the return dict should be of type str. Got {type(key)}.')
                     _validate_single_metric(value)
-
-                if isinstance(direction, dict):
-                    for direction_key in direction:
-                        assert direction_key in result, \
-                            (f'tensorleap_custom_metric validation failed: '
-                             f'Keys in the direction mapping should be part of result keys. Got key {direction_key}.')
-
-                if isinstance(compute_insights, dict):
-                    for ci_key in compute_insights:
-                        assert ci_key in result, \
-                            (f'tensorleap_custom_metric validation failed: '
-                             f'Keys in the compute_insights mapping should be part of result keys. Got key {ci_key}.')
-
             else:
                 _validate_single_metric(result)
 

code_loader/leaploader.py

@@ -14,7 +14,7 @@ import numpy.typing as npt
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
-    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap
+    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -233,9 +233,19 @@ class LeapLoader(LeapLoaderBase):
         self._preprocess_result()
         return self._metric_handler_by_name()[metric_name].function(**input_tensors_by_arg_name)
 
-    def run_custom_loss(self, custom_loss_name: str,
+    def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
-        return self._custom_loss_handler_by_name()[custom_loss_name].function(**input_tensors_by_arg_name)
+
+        custom_loss_handler = self._custom_loss_handler_by_name()[custom_loss_name]
+        preprocess_response_arg_name = None
+        for arg_name, arg_type in inspect.getfullargspec(custom_loss_handler.function).annotations.items():
+            if arg_type == SamplePreprocessResponse:
+                preprocess_response_arg_name = arg_name
+                break
+
+        if preprocess_response_arg_name is not None:
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(sample_ids, self._preprocess_result()[state])
+        return custom_loss_handler.function(**input_tensors_by_arg_name)
 
     def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
         # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
@@ -320,7 +330,8 @@ class LeapLoader(LeapLoaderBase):
             for visualizer_handler in setup.visualizers]
 
         custom_losses = [CustomLossInstance(custom_loss.custom_loss_handler_data.name,
-                                            custom_loss.custom_loss_handler_data.arg_names)
+                                            custom_loss.custom_loss_handler_data.arg_names,
+                                            custom_loss.custom_loss_handler_data.preprocess_response_arg_name)
                          for custom_loss in setup.custom_loss_handlers]
 
         prediction_types = []

code_loader/leaploaderbase.py

@@ -20,6 +20,9 @@ class LeapLoaderBase:
         self.code_entry_name = code_entry_name
         self.code_path = code_path
 
+        self.current_working_sample_ids: Optional[np.array] = None
+        self.current_working_state: Optional[DataStateEnum] = None
+
     @abstractmethod
     def metric_by_name(self) -> Dict[str, MetricHandlerData]:
         pass
@@ -58,7 +61,7 @@ class LeapLoaderBase:
         pass
 
     @abstractmethod
-    def run_custom_loss(self, custom_loss_name: str,
+    def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
         pass
 

{code_loader-1.0.72.dist-info → code_loader-1.0.72.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.72
+Version: 1.0.72.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.72.dist-info → code_loader-1.0.72.dev1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=vnxcit0LKNdXAMQi4EhjDiUpwsavAw2YMcEwNwmCsJo,7063
+code_loader/contract/datasetclasses.py,sha256=2m9wyF_SO_q3kfrrAELS8rseMX-veeaP-Aof0ZhG_7g,7119
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=RSx9m_R3LawhK5o1nAcO3hfp2F9oJYtxZr_bpP3bTmw,4005
@@ -19,14 +19,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=qBpN_hNANcXi6ilPBauBBWSmMt3tl4Ha5JkLe1dPzjE,26571
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=B-XSw4xYF39kMPnMTRNKMYFg09whnfl7VSbcx195VG8,21626
-code_loader/leaploader.py,sha256=GWlpvgSsCWevP2BwwFBKTImQeDgHAQg1lMU9bqFMwRw,22315
-code_loader/leaploaderbase.py,sha256=aHlqWDZRacIdBefeB9goYVnpApaNN2FT24uPIWKkCeQ,3090
+code_loader/inner_leap_binder/leapbinder.py,sha256=una-6k_nnSeqrLNjCy3beXdje_MWXbCRLoCalKdNYbg,26693
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=ebMxknpKMW-dE8Erq0fFq4RrE5E_Jfx9IvmRRZSdhlc,20813
+code_loader/leaploader.py,sha256=DCrSS2789dkkePROJSwhAhgzQvR4_27YAR__PxCcLNc,23025
+code_loader/leaploaderbase.py,sha256=FDXjTFBjnpfUyZ2tS3wLo7y2j82Qb_VKHI1BMB5gFww,3269
 code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=Ffx5VHVOe5ujBOsjBSxN_aIEVwFSQ6gbhTMG5aUS-po,2305
-code_loader-1.0.72.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.72.dist-info/METADATA,sha256=YRI0I_t6gfDnauY5w93AZSY1aAILsGSJujWfEFxErY4,849
-code_loader-1.0.72.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.72.dist-info/RECORD,,
+code_loader-1.0.72.dev1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.72.dev1.dist-info/METADATA,sha256=WBFziucTTs-QHppp3QzdoSp_j9KXh1W9njNsHAGxxSo,854
+code_loader-1.0.72.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.72.dev1.dist-info/RECORD,,