code-loader 1.0.72.dev4__py3-none-any.whl → 1.0.72.dev5__py3-none-any.whl

code_loader/leaploader.py

@@ -6,7 +6,7 @@ import sys
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union, Any, Type, Optional
+from typing import Dict, List, Iterable, Union, Any, Type, Optional, Callable
 
 import numpy as np
 import numpy.typing as npt
@@ -213,7 +213,8 @@ class LeapLoader(LeapLoaderBase):
                         preprocess_response, test_result, dataset_base_handler)
                 except Exception as e:
                     line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
-                    test_result[0].display[state_name] = f"{repr(e)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
+                    test_result[0].display[
+                        state_name] = f"{repr(e)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                     test_result[0].is_passed = False
 
             result_payloads.extend(test_result)
@@ -228,39 +229,69 @@ class LeapLoader(LeapLoaderBase):
         all_dataset_base_handlers.extend(global_leap_binder.setup_container.metadata)
         return all_dataset_base_handlers
 
-    def run_metric(self, metric_name: str,
+    def run_metric(self, metric_name: str, sample_ids: np.array, state: DataStateEnum,
                    input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
         self._preprocess_result()
-        return self._metric_handler_by_name()[metric_name].function(**input_tensors_by_arg_name)
+
+        metric_handler = self._metric_handler_by_name()[metric_name]
+        preprocess_response_arg_name = self._get_preprocess_response_arg_name(metric_handler.function)
+
+        if preprocess_response_arg_name is not None:
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(
+                sample_ids, self._preprocess_result()[state])
+
+        return metric_handler.function(**input_tensors_by_arg_name)
+
+    @staticmethod
+    def _get_preprocess_response_arg_name(
+            func: Callable) -> Optional[str]:
+        for arg_name, arg_type in inspect.getfullargspec(func).annotations.items():
+            if arg_type == SamplePreprocessResponse:
+                return arg_name
+        return None
 
     def run_custom_loss(self, custom_loss_name: str, sample_ids: np.array, state: DataStateEnum,
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
 
         custom_loss_handler = self._custom_loss_handler_by_name()[custom_loss_name]
-        preprocess_response_arg_name = None
-        for arg_name, arg_type in inspect.getfullargspec(custom_loss_handler.function).annotations.items():
-            if arg_type == SamplePreprocessResponse:
-                preprocess_response_arg_name = arg_name
-                break
+        preprocess_response_arg_name = self._get_preprocess_response_arg_name(custom_loss_handler.function)
 
         if preprocess_response_arg_name is not None:
-            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(sample_ids, self._preprocess_result()[state])
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(sample_ids,
+                                                                                               self._preprocess_result()[
+                                                                                                   state])
         return custom_loss_handler.function(**input_tensors_by_arg_name)
 
-    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
+    def run_visualizer(self, visualizer_name: str, sample_ids: np.array, state: DataStateEnum,
+                       input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
         # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
         # global param that visualizer is using)
         self._preprocess_result()
 
-        return self._visualizer_handler_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
+        vis_handler = self._visualizer_handler_by_name()[visualizer_name]
+        preprocess_response_arg_name = self._get_preprocess_response_arg_name(vis_handler.function)
+
+        if preprocess_response_arg_name is not None:
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(sample_ids,
+                                                                                               self._preprocess_result()[
+                                                                                                   state])
+
+        return vis_handler.function(**input_tensors_by_arg_name)
 
-    def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+    def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
+                               sample_ids: np.array, state: DataStateEnum,
                                ) -> Optional[npt.NDArray[np.float32]]:
         heatmap_function = self._visualizer_handler_by_name()[visualizer_name].heatmap_function
         if heatmap_function is None:
             assert len(input_tensors_by_arg_name) == 1
             return None
 
+        preprocess_response_arg_name = self._get_preprocess_response_arg_name(heatmap_function)
+        if preprocess_response_arg_name is not None:
+            input_tensors_by_arg_name[preprocess_response_arg_name] = SamplePreprocessResponse(sample_ids,
+                                                                                               self._preprocess_result()[
+                                                                                                   state])
+
         return heatmap_function(**input_tensors_by_arg_name)
 
     def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
@@ -309,7 +340,8 @@ class LeapLoader(LeapLoaderBase):
             if hasattr(handler_test_payload.raw_result, 'tolist'):
                 handler_test_payload.raw_result = handler_test_payload.raw_result.tolist()
             metadata_type = type(handler_test_payload.raw_result)
-            if metadata_type == int or isinstance(handler_test_payload.raw_result, (np.unsignedinteger, np.signedinteger)):
+            if metadata_type == int or isinstance(handler_test_payload.raw_result,
+                                                  (np.unsignedinteger, np.signedinteger)):
                 metadata_type = float
             if isinstance(handler_test_payload.raw_result, str):
                 dataset_metadata_type = DatasetMetadataType.string
@@ -369,7 +401,8 @@ class LeapLoader(LeapLoaderBase):
 
         return self._preprocess_result_cached
 
-    def get_preprocess_sample_ids(self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
+    def get_preprocess_sample_ids(self, update_unlabeled_preprocess=False) -> Dict[
+        DataStateEnum, Union[List[int], List[str]]]:
         preprocess_result = self._preprocess_result(update_unlabeled_preprocess)
         sample_ids = {}
         for state, preprocess_response in preprocess_result.items():
@@ -427,7 +460,8 @@ class LeapLoader(LeapLoaderBase):
 
         return converted_value
 
-    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, Union[str, int, bool, float]]:
+    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[
+        str, Union[str, int, bool, float]]:
         result_agg = {}
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
@@ -436,7 +470,8 @@ class LeapLoader(LeapLoaderBase):
             if isinstance(handler_result, dict):
                 for single_metadata_name, single_metadata_result in handler_result.items():
                     handler_name = f'{handler.name}_{single_metadata_name}'
-                    result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name, single_metadata_result)
+                    result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name,
+                                                                                      single_metadata_result)
             else:
                 handler_name = handler.name
                 result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name, handler_result)
@@ -452,5 +487,3 @@ class LeapLoader(LeapLoaderBase):
                 raise Exception("Different id types in preprocess results")
 
         return id_type
-
-

code_loader/leaploaderbase.py

@@ -69,11 +69,12 @@ class LeapLoaderBase:
         pass
 
     @abstractmethod
-    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
+    def run_visualizer(self, visualizer_name: str, sample_ids: np.array, state: DataStateEnum,
+                       input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
         pass
 
     @abstractmethod
-    def run_metric(self, metric_name: str,
+    def run_metric(self, metric_name: str, sample_ids: np.array, state: DataStateEnum,
                    input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
         pass
 
@@ -83,7 +84,8 @@ class LeapLoaderBase:
         pass
 
     @abstractmethod
-    def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+    def run_heatmap_visualizer(self, visualizer_name: str, sample_ids: np.array, state: DataStateEnum,
+                               input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
                                ) -> Optional[npt.NDArray[np.float32]]:
         pass
 

{code_loader-1.0.72.dev4.dist-info → code_loader-1.0.72.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.72.dev4
+Version: 1.0.72.dev5
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.72.dev4.dist-info → code_loader-1.0.72.dev5.dist-info}/RECORD

@@ -21,12 +21,12 @@ code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaS
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
 code_loader/inner_leap_binder/leapbinder.py,sha256=-fryKzD8T8K2EgrOsR5NryabP8_1k_m3POLwhYIA_8I,26708
 code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=ebMxknpKMW-dE8Erq0fFq4RrE5E_Jfx9IvmRRZSdhlc,20813
-code_loader/leaploader.py,sha256=g160Z0MRop_7m3bu1HawcGoZNKakvs9Su4tEfbQ9pR0,22914
-code_loader/leaploaderbase.py,sha256=ijTodEBL-Q9DulR9z0xU0fo72rVvm06VvoRyrxzoCtE,4012
+code_loader/leaploader.py,sha256=he1c46jQNrCBEBq03gQDS0WVxiX7nlGAE9_9hLagxQc,24973
+code_loader/leaploaderbase.py,sha256=VH0vddRmkqLtcDlYPCO7hfz1_VbKo43lUdHDAbd4iJc,4198
 code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=Ffx5VHVOe5ujBOsjBSxN_aIEVwFSQ6gbhTMG5aUS-po,2305
-code_loader-1.0.72.dev4.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.72.dev4.dist-info/METADATA,sha256=TQ8fEw3bxJr1ZLchIY_e8EMfRVWreauzQhnK426Ca2k,854
-code_loader-1.0.72.dev4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.72.dev4.dist-info/RECORD,,
+code_loader-1.0.72.dev5.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.72.dev5.dist-info/METADATA,sha256=IuNmOiTSxEC2A_-cCOc06mXUHiZPaf-Wg71eT28YGJE,854
+code_loader-1.0.72.dev5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.72.dev5.dist-info/RECORD,,