code-loader 1.0.61.dev2__py3-none-any.whl → 1.0.61.dev4__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -124,24 +124,34 @@ class CustomLossHandler:
 
 
 @dataclass
-class MetricHandler:
+class MetricHandlerData:
     name: str
-    function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
     arg_names: List[str]
     direction: Optional[MetricDirection] = MetricDirection.Downward
 
 
+@dataclass
+class MetricHandler:
+    metric_handler_data: MetricHandlerData
+    function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
+
+
 @dataclass
 class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
 
 
 @dataclass
-class VisualizerHandler:
+class VisualizerHandlerData:
     name: str
-    function: VisualizerCallableInterface
     type: LeapDataType
     arg_names: List[str]
+
+
+@dataclass
+class VisualizerHandler:
+    visualizer_handler_data: VisualizerHandlerData
+    function: VisualizerCallableInterface
     heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None
 
 

code_loader/inner_leap_binder/leapbinder.py

@@ -9,7 +9,8 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
-    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, RawInputsForHeatmap
+    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
+    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
@@ -127,7 +128,7 @@ class LeapBinder:
                     f'should be {expected_return_type}')
 
         self.setup_container.visualizers.append(
-            VisualizerHandler(name, function, visualizer_type, arg_names, heatmap_visualizer))
+            VisualizerHandler(VisualizerHandlerData(name, visualizer_type, arg_names), function, heatmap_visualizer))
         self._visualizer_names.append(name)
 
     def set_preprocess(self, function: Callable[[], List[PreprocessResponse]]) -> None:
@@ -251,7 +252,7 @@ class LeapBinder:
             leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
         """
         arg_names = inspect.getfullargspec(function)[0]
-        self.setup_container.metrics.append(MetricHandler(name, function, arg_names, direction))
+        self.setup_container.metrics.append(MetricHandler(MetricHandlerData(name, arg_names, direction), function))
 
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1) -> None:
         """

code_loader/leaploader.py

@@ -12,7 +12,8 @@ import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
     PreprocessResponse, VisualizerHandler, LeapData, CustomLossHandler, \
-    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler
+    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
+    MetricCallableReturnType
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -65,7 +66,16 @@ class LeapLoader(LeapLoaderBase):
         spec.loader.exec_module(file)
 
     @lru_cache()
-    def metric_by_name(self) -> Dict[str, MetricHandler]:
+    def metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            metric_handler.name: metric_handler.metric_handler_data
+            for metric_handler in setup.metrics
+        }
+
+    @lru_cache()
+    def _metric_handler_by_name(self) -> Dict[str, MetricHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
@@ -74,7 +84,16 @@ class LeapLoader(LeapLoaderBase):
         }
 
     @lru_cache()
-    def visualizer_by_name(self) -> Dict[str, VisualizerHandler]:
+    def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            visualizer_handler.name: visualizer_handler.visualizer_handler_data
+            for visualizer_handler in setup.visualizers
+        }
+
+    @lru_cache()
+    def _visualizer_handler_by_name(self) -> Dict[str, VisualizerHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
@@ -199,17 +218,21 @@ class LeapLoader(LeapLoaderBase):
         all_dataset_base_handlers.extend(global_leap_binder.setup_container.metadata)
         return all_dataset_base_handlers
 
-    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
-                       ) -> LeapData:
+    def run_metric(self, metric_name: str,
+                   input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
+        self._preprocess_result()
+        return self._metric_handler_by_name()[metric_name].function(**input_tensors_by_arg_name)
+
+    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
         # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
         # global param that visualizer is using)
         self._preprocess_result()
 
-        return self.visualizer_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
+        return self._visualizer_handler_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
 
     def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
                                ) -> npt.NDArray[np.float32]:
-        heatmap_function = self.visualizer_by_name()[visualizer_name].heatmap_function
+        heatmap_function = self._visualizer_handler_by_name()[visualizer_name].heatmap_function
         if heatmap_function is None:
             assert len(input_tensors_by_arg_name) == 1
             return list(input_tensors_by_arg_name.values())[0]

code_loader/leaploaderbase.py

@@ -7,8 +7,8 @@ from typing import Dict, List, Union, Type
 import numpy as np
 import numpy.typing as npt
 
-from code_loader.contract.datasetclasses import DatasetSample, VisualizerHandler, LeapData, CustomLossHandler, \
-    PredictionTypeHandler, CustomLayerHandler, MetricHandler
+from code_loader.contract.datasetclasses import DatasetSample, LeapData, CustomLossHandler, \
+    PredictionTypeHandler, CustomLayerHandler, VisualizerHandlerData, MetricHandlerData, MetricCallableReturnType
 from code_loader.contract.enums import DataStateEnum
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetSetup, ModelSetup
@@ -20,11 +20,11 @@ class LeapLoaderBase:
         self.code_path = code_path
 
     @abstractmethod
-    def metric_by_name(self) -> Dict[str, MetricHandler]:
+    def metric_by_name(self) -> Dict[str, MetricHandlerData]:
         pass
 
     @abstractmethod
-    def visualizer_by_name(self) -> Dict[str, VisualizerHandler]:
+    def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
         pass
 
     @abstractmethod
@@ -48,8 +48,12 @@ class LeapLoaderBase:
         pass
 
     @abstractmethod
-    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
-                       ) -> LeapData:
+    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
+        pass
+
+    @abstractmethod
+    def run_metric(self, metric_name: str,
+                   input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
         pass
 
     @abstractmethod

{code_loader-1.0.61.dev2.dist-info → code_loader-1.0.61.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.61.dev2
+Version: 1.0.61.dev4
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.61.dev2.dist-info → code_loader-1.0.61.dev4.dist-info}/RECORD

@@ -2,7 +2,7 @@ LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
 code_loader/code_inegration_processes_manager.py,sha256=XslWOPeNQk4RAFJ_f3tP5Oe3EgcIR7BE7Y8r9Ty73-o,3261
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=lFS7_weizsjzx4_tYwYGrrRUj1sgIl010h9FON4brb8,6670
+code_loader/contract/datasetclasses.py,sha256=LVsZIPpxgjz9ycYU21nNe2-7ug-A8rpi-X236m-2HwM,6844
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
@@ -18,14 +18,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=35hyesDdmjOD9wdrTLyayb-vm9aDfmEbMA0c4EQR1LA,25090
+code_loader/inner_leap_binder/leapbinder.py,sha256=bb-z_QS3b1eQc2Be0lp5CDm8eRCP9NN1f7Hu2pQe_4E,25180
 code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=uuM_ht9HZ1GH2IabKeGQ_x9NmD3poK_h1Gt0NruwJuY,19704
-code_loader/leaploader.py,sha256=KC_6oso5pbOHZ56sUTcV6qdFzEbIJ8MdEtKu-nDrQfE,19707
-code_loader/leaploaderbase.py,sha256=ZPncue31Ld6NeaOZz4H0PJXxl5AYJfC01tX_H_ARVFc,2542
+code_loader/leaploader.py,sha256=uk2E1_0psS89W68y_pTId3ta7d1s8gmJXDbAzt1sqzw,20708
+code_loader/leaploaderbase.py,sha256=Ursxo27dhiWUc5kG0WcQwJPkcwBFaq6U3sn8DHDLOdg,2747
 code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=VoqO9FN84yXyMjRjHjUTOt2GdTkJRMbHbXJ1cJkREkk,2230
-code_loader-1.0.61.dev2.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.61.dev2.dist-info/METADATA,sha256=EzpmH_OVUl7AVQxhi2-Gth86YjYw9N0PdIKDl_kd1J4,893
-code_loader-1.0.61.dev2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.61.dev2.dist-info/RECORD,,
+code_loader-1.0.61.dev4.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.61.dev4.dist-info/METADATA,sha256=aolebyNYL9I7vdmTng-2XkEBSCijaCV1p9nQ0aQLEU4,893
+code_loader-1.0.61.dev4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.61.dev4.dist-info/RECORD,,