code-loader 1.0.61.dev6__tar.gz → 1.0.63__tar.gz

{code_loader-1.0.61.dev6 → code_loader-1.0.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.61.dev6
+Version: 1.0.63
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT
@@ -13,7 +13,6 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: matplotlib (>=3.3,<3.4)
 Requires-Dist: numpy (>=1.22.3,<2.0.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)

{code_loader-1.0.61.dev6 → code_loader-1.0.63}/code_loader/contract/datasetclasses.py

@@ -116,29 +116,19 @@ ConfusionMatrixCallableInterfaceMultiArgs = Callable[..., List[List[ConfusionMat
 MetricCallableReturnType = Union[Any, List[List[ConfusionMatrixElement]]]
 
 
-@dataclass
-class CustomLossHandlerData:
-    name: str
-    arg_names: List[str]
-
-
 @dataclass
 class CustomLossHandler:
-    custom_loss_handler_data: CustomLossHandlerData
-    function: CustomCallableInterface
-
-
-@dataclass
-class MetricHandlerData:
     name: str
+    function: CustomCallableInterface
     arg_names: List[str]
-    direction: Optional[MetricDirection] = MetricDirection.Downward
 
 
 @dataclass
 class MetricHandler:
-    metric_handler_data: MetricHandlerData
+    name: str
     function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
+    arg_names: List[str]
+    direction: Optional[MetricDirection] = MetricDirection.Downward
 
 
 @dataclass
@@ -147,16 +137,11 @@ class RawInputsForHeatmap:
 
 
 @dataclass
-class VisualizerHandlerData:
+class VisualizerHandler:
     name: str
+    function: VisualizerCallableInterface
     type: LeapDataType
     arg_names: List[str]
-
-
-@dataclass
-class VisualizerHandler:
-    visualizer_handler_data: VisualizerHandlerData
-    function: VisualizerCallableInterface
     heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None
 
 
@@ -169,7 +154,7 @@ class DatasetBaseHandler:
 @dataclass
 class InputHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
-
+    channel_dim: Optional[int] = -1
 
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):

{code_loader-1.0.61.dev6 → code_loader-1.0.63}/code_loader/contract/responsedataclasses.py

@@ -20,6 +20,7 @@ class DatasetBaseSectionInstance:
 @dataclass
 class DatasetInputInstance(DatasetBaseSectionInstance):
     shape: List[int]
+    channel_dim: Optional[int] = -1
 
 
 @dataclass

{code_loader-1.0.61.dev6 → code_loader-1.0.63}/code_loader/inner_leap_binder/leapbinder.py

@@ -9,8 +9,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
-    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
-    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData
+    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, RawInputsForHeatmap
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
@@ -128,7 +127,7 @@ class LeapBinder:
                     f'should be {expected_return_type}')
 
         self.setup_container.visualizers.append(
-            VisualizerHandler(VisualizerHandlerData(name, visualizer_type, arg_names), function, heatmap_visualizer))
+            VisualizerHandler(name, function, visualizer_type, arg_names, heatmap_visualizer))
         self._visualizer_names.append(name)
 
     def set_preprocess(self, function: Callable[[], List[PreprocessResponse]]) -> None:
@@ -182,13 +181,14 @@ class LeapBinder:
         """
         self.setup_container.unlabeled_data_preprocess = UnlabeledDataPreprocessHandler(function)
 
-    def set_input(self, function: SectionCallableInterface, name: str) -> None:
+    def set_input(self, function: SectionCallableInterface, name: str, channel_dim: int = -1) -> None:
         """
         Set the input handler function.
 
         Args:
         function (SectionCallableInterface): The input handler function.
         name (str): The name of the input section.
+        channel_dim (int): The dimension of the channels axis
 
         Example:
             def input_encoder(subset: PreprocessResponse, index: int) -> np.ndarray:
@@ -198,10 +198,10 @@ class LeapBinder:
                 img = normalize(img)
                 return img
 
-            leap_binder.set_input(input_encoder, name='input_encoder')
+            leap_binder.set_input(input_encoder, name='input_encoder', channel_dim=-1)
         """
         function = to_numpy_return_wrapper(function)
-        self.setup_container.inputs.append(InputHandler(name, function))
+        self.setup_container.inputs.append(InputHandler(name, function, channel_dim=channel_dim))
 
         self._encoder_names.append(name)
 
@@ -226,7 +226,7 @@ class LeapBinder:
             leap_binder.add_custom_loss(custom_loss_function, name='custom_loss')
         """
         arg_names = inspect.getfullargspec(function)[0]
-        self.setup_container.custom_loss_handlers.append(CustomLossHandler(CustomLossHandlerData(name, arg_names), function))
+        self.setup_container.custom_loss_handlers.append(CustomLossHandler(name, function, arg_names))
 
     def add_custom_metric(self,
                           function: Union[CustomCallableInterfaceMultiArgs,
@@ -252,7 +252,7 @@ class LeapBinder:
             leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
         """
         arg_names = inspect.getfullargspec(function)[0]
-        self.setup_container.metrics.append(MetricHandler(MetricHandlerData(name, arg_names, direction), function))
+        self.setup_container.metrics.append(MetricHandler(name, function, arg_names, direction))
 
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1) -> None:
         """

{code_loader-1.0.61.dev6 → code_loader-1.0.63}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -58,11 +58,13 @@ def tensorleap_custom_metric(name: str, direction: Optional[MetricDirection] = M
                         assert isinstance(single_metric_result[0][0], ConfusionMatrixElement), \
                             f'{supported_types_message}Got List[List[{type(single_metric_result[0][0])}]].'
                     else:
-                        assert isinstance(single_metric_result[0], float), f'{supported_types_message}Got List[{type(single_metric_result[0])}].'
+                        assert isinstance(single_metric_result[0],
+                                          float), f'{supported_types_message}Got List[{type(single_metric_result[0])}].'
                 else:
-                    assert isinstance(single_metric_result, np.ndarray), f'{supported_types_message}Got {type(single_metric_result)}.'
+                    assert isinstance(single_metric_result,
+                                      np.ndarray), f'{supported_types_message}Got {type(single_metric_result)}.'
                     assert len(single_metric_result.shape) == 1, (f'tensorleap_custom_metric validation failed: '
-                                                    f'The return shape should be 1D. Got {len(single_metric_result.shape)}D.')
+                                                                  f'The return shape should be 1D. Got {len(single_metric_result.shape)}D.')
 
                 if leap_binder.batch_size_to_validate:
                     assert len(single_metric_result) == leap_binder.batch_size_to_validate, \
@@ -245,14 +247,16 @@ def tensorleap_unlabeled_preprocess():
     return decorating_function
 
 
-def tensorleap_input_encoder(name: str):
+def tensorleap_input_encoder(name: str, channel_dim=-1):
     def decorating_function(user_function: SectionCallableInterface):
         for input_handler in leap_binder.setup_container.inputs:
             if input_handler.name == name:
                 raise Exception(f'Input with name {name} already exists. '
                                 f'Please choose another')
+        if channel_dim <= 0 and channel_dim != -1:
+            raise Exception(f"Channel dim for input {name} is expected to be either -1 or positive")
 
-        leap_binder.set_input(user_function, name)
+        leap_binder.set_input(user_function, name, channel_dim=channel_dim)
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
             assert isinstance(sample_id, (int, str)), \
@@ -273,6 +277,8 @@ def tensorleap_input_encoder(name: str):
             assert result.dtype == np.float32, \
                 (f'tensorleap_input_encoder validation failed: '
                  f'The return type should be a numpy array of type float32. Got {result.dtype}.')
+            assert channel_dim - 1 <= len(result.shape), (f'tensorleap_input_encoder validation failed: '
+                                                          f'The channel_dim ({channel_dim}) should be <= to the rank of the resulting input rank ({len(result.shape)}).')
 
         def inner(sample_id, preprocess_response):
             _validate_input_args(sample_id, preprocess_response)
@@ -341,11 +347,21 @@ def tensorleap_custom_loss(name: str):
                 raise Exception('the input arguments of the custom loss function should be tensorflow tensors') from e
 
             for i, arg in enumerate(args):
-                assert isinstance(arg, tf.Tensor), (f'tensorleap_custom_loss validation failed: '
-                                                    f'Argument #{i} should be a tensorflow tensor. Got {type(arg)}.')
+                if isinstance(arg, list):
+                    for y, elem in enumerate(arg):
+                        assert isinstance(elem, tf.Tensor), (f'tensorleap_custom_loss validation failed: '
+                                                             f'Element #{y} of list should be a tensorflow tensor. Got {type(elem)}.')
+                else:
+                    assert isinstance(arg, tf.Tensor), (f'tensorleap_custom_loss validation failed: '
+                                                        f'Argument #{i} should be a tensorflow tensor. Got {type(arg)}.')
             for _arg_name, arg in kwargs.items():
-                assert isinstance(arg, tf.Tensor), (f'tensorleap_custom_loss validation failed: '
-                                                    f'Argument {_arg_name} should be a tensorflow tensor. Got {type(arg)}.')
+                if isinstance(arg, list):
+                    for y, elem in enumerate(arg):
+                        assert isinstance(elem, tf.Tensor), (f'tensorleap_custom_loss validation failed: '
+                                                             f'Element #{y} of list should be a tensorflow tensor. Got {type(elem)}.')
+                else:
+                    assert isinstance(arg, tf.Tensor), (f'tensorleap_custom_loss validation failed: '
+                                                        f'Argument #{_arg_name} should be a tensorflow tensor. Got {type(arg)}.')
 
         def _validate_result(result):
             try:

{code_loader-1.0.61.dev6 → code_loader-1.0.63}/code_loader/leaploader.py

@@ -2,6 +2,7 @@
 import importlib.util
 import io
 import sys
+import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
@@ -11,22 +12,21 @@ import numpy as np
 import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
-    PreprocessResponse, VisualizerHandler, LeapData, \
-    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
-    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler
+    PreprocessResponse, VisualizerHandler, LeapData, CustomLossHandler, \
+    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance
 from code_loader.inner_leap_binder import global_leap_binder
-from code_loader.leaploaderbase import LeapLoaderBase
 from code_loader.utils import get_root_exception_file_and_line_number
 
 
-class LeapLoader(LeapLoaderBase):
+class LeapLoader:
     def __init__(self, code_path: str, code_entry_name: str):
-        super().__init__(code_path, code_entry_name)
+        self.code_entry_name = code_entry_name
+        self.code_path = code_path
 
         self._preprocess_result_cached = None
 
@@ -66,16 +66,7 @@ class LeapLoader(LeapLoaderBase):
         spec.loader.exec_module(file)
 
     @lru_cache()
-    def metric_by_name(self) -> Dict[str, MetricHandlerData]:
-        self.exec_script()
-        setup = global_leap_binder.setup_container
-        return {
-            metric_handler.name: metric_handler.metric_handler_data
-            for metric_handler in setup.metrics
-        }
-
-    @lru_cache()
-    def _metric_handler_by_name(self) -> Dict[str, MetricHandler]:
+    def metric_by_name(self) -> Dict[str, MetricHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
@@ -84,16 +75,7 @@ class LeapLoader(LeapLoaderBase):
         }
 
     @lru_cache()
-    def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
-        self.exec_script()
-        setup = global_leap_binder.setup_container
-        return {
-            visualizer_handler.name: visualizer_handler.visualizer_handler_data
-            for visualizer_handler in setup.visualizers
-        }
-
-    @lru_cache()
-    def _visualizer_handler_by_name(self) -> Dict[str, VisualizerHandler]:
+    def visualizer_by_name(self) -> Dict[str, VisualizerHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
@@ -102,16 +84,7 @@ class LeapLoader(LeapLoaderBase):
         }
 
     @lru_cache()
-    def custom_loss_by_name(self) -> Dict[str, CustomLossHandlerData]:
-        self.exec_script()
-        setup = global_leap_binder.setup_container
-        return {
-            custom_loss_handler.name: custom_loss_handler.custom_loss_handler_data
-            for custom_loss_handler in setup.custom_loss_handlers
-        }
-
-    @lru_cache()
-    def _custom_loss_handler_by_name(self) -> Dict[str, CustomLossHandler]:
+    def custom_loss_by_name(self) -> Dict[str, CustomLossHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
@@ -227,31 +200,20 @@ class LeapLoader(LeapLoaderBase):
         all_dataset_base_handlers.extend(global_leap_binder.setup_container.metadata)
         return all_dataset_base_handlers
 
-    def run_metric(self, metric_name: str,
-                   input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
-        self._preprocess_result()
-        return self._metric_handler_by_name()[metric_name].function(**input_tensors_by_arg_name)
-
-    def run_custom_loss(self, custom_loss_name: str,
-                        input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
-        return self._custom_loss_handler_by_name()[custom_loss_name].function(**input_tensors_by_arg_name)
-
-    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
-        # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
-        # global param that visualizer is using)
-        self._preprocess_result()
-
-        return self._visualizer_handler_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
+    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
+                       ) -> LeapData:
+        return self.visualizer_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
 
     def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
                                ) -> npt.NDArray[np.float32]:
-        heatmap_function = self._visualizer_handler_by_name()[visualizer_name].heatmap_function
+        heatmap_function = self.visualizer_by_name()[visualizer_name].heatmap_function
         if heatmap_function is None:
             assert len(input_tensors_by_arg_name) == 1
             return list(input_tensors_by_arg_name.values())[0]
         return heatmap_function(**input_tensors_by_arg_name)
 
-    def get_dataset_setup_response(self, handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
+    @staticmethod
+    def get_dataset_setup_response(handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
         setup = global_leap_binder.setup_container
         assert setup.preprocess is not None
 
@@ -269,7 +231,7 @@ class LeapLoader(LeapLoaderBase):
         for inp in setup.inputs:
             if inp.shape is None:
                 raise Exception(f"cant calculate shape for input, input name:{inp.name}")
-            inputs.append(DatasetInputInstance(name=inp.name, shape=inp.shape))
+            inputs.append(DatasetInputInstance(name=inp.name, shape=inp.shape, channel_dim=inp.channel_dim))
 
         ground_truths = []
         for gt in setup.ground_truths:
@@ -321,7 +283,8 @@ class LeapLoader(LeapLoaderBase):
                             metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,
                             custom_losses=custom_losses, metrics=metrics)
 
-    def get_model_setup_response(self) -> ModelSetup:
+    @staticmethod
+    def get_model_setup_response() -> ModelSetup:
         setup = global_leap_binder.setup_container
         custom_layer_instances = [
             CustomLayerInstance(custom_layer_handler.name, custom_layer_handler.init_arg_names,

{code_loader-1.0.61.dev6 → code_loader-1.0.63}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.61.dev6"
+version = "1.0.63"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"
@@ -15,7 +15,6 @@ include = [
 python = ">=3.8,<3.12"
 numpy = "^1.22.3"
 psutil = "^5.9.5"
-matplotlib = ">=3.3,<3.4"
 requests = "^2.32.3"
 pyyaml = "^6.0.2"
 

code_loader-1.0.61.dev6/code_loader/code_inegration_processes_manager.py

@@ -1,83 +0,0 @@
-# mypy: ignore-errors
-import traceback
-from dataclasses import dataclass
-
-from typing import List, Tuple, Optional
-
-from multiprocessing import Process, Queue
-
-from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
-from code_loader.leaploader import LeapLoader
-from code_loader.contract.enums import DataStateEnum
-from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
-from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
-
-
-@dataclass
-class SampleSerializableError:
-    state: DataStateEnum
-    index: int
-    leap_script_trace: str
-    exception_as_str: str
-
-
-class CodeIntegrationProcessesManager:
-    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
-                 max_samples_in_queue: int = 128) -> None:
-        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
-        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
-
-    def _create_and_start_process(self) -> Process:
-        process = self.multiprocessing_context.Process(
-            target=CodeIntegrationProcessesManager._process_func,
-            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
-                  self._ready_processed_results))
-        process.daemon = True
-        process.start()
-        return process
-
-    def _run_and_warm_first_process(self):
-        process = self._create_and_start_process()
-        self.processes = [process]
-
-        # needed in order to make sure the preprocess func runs once in nonparallel
-        self._start_process_inputs([(DataStateEnum.training, 0)])
-        self._get_next_ready_processed_result()
-
-    def _operation_decider(self):
-        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
-            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'metric'
-
-        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
-            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'dataset'
-
-
-
-
-    @staticmethod
-    def _process_func(code_path: str, code_entry_name: str,
-                      samples_to_process: Queue, ready_samples: Queue,
-                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
-        import os
-        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
-
-        leap_loader = LeapLoader(code_path, code_entry_name)
-        while True:
-
-            # decide on sample or metric to process
-            state, idx = samples_to_process.get(block=True)
-            leap_loader._preprocess_result()
-            try:
-                sample = leap_loader.get_sample(state, idx)
-            except Exception as e:
-                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
-                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
-                continue
-
-            ready_samples.put(sample)
-
-    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
-        return self.start_process_inputs(sample_identities)
-

code_loader-1.0.61.dev6/code_loader/leaploaderbase.py

@@ -1,91 +0,0 @@
-# mypy: ignore-errors
-
-from abc import abstractmethod
-
-from typing import Dict, List, Union, Type
-
-import numpy as np
-import numpy.typing as npt
-
-from code_loader.contract.datasetclasses import DatasetSample, LeapData, \
-    PredictionTypeHandler, CustomLayerHandler, VisualizerHandlerData, MetricHandlerData, MetricCallableReturnType, \
-    CustomLossHandlerData
-from code_loader.contract.enums import DataStateEnum
-from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
-    DatasetSetup, ModelSetup
-
-
-class LeapLoaderBase:
-    def __init__(self, code_path: str, code_entry_name: str):
-        self.code_entry_name = code_entry_name
-        self.code_path = code_path
-
-    @abstractmethod
-    def metric_by_name(self) -> Dict[str, MetricHandlerData]:
-        pass
-
-    @abstractmethod
-    def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
-        pass
-
-    @abstractmethod
-    def custom_loss_by_name(self) -> Dict[str, CustomLossHandlerData]:
-        pass
-
-    @abstractmethod
-    def custom_layers(self) -> Dict[str, CustomLayerHandler]:
-        pass
-
-    @abstractmethod
-    def prediction_type_by_name(self) -> Dict[str, PredictionTypeHandler]:
-        pass
-
-    @abstractmethod
-    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
-        pass
-
-    @abstractmethod
-    def check_dataset(self) -> DatasetIntegParseResult:
-        pass
-
-    @abstractmethod
-    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
-        pass
-
-    @abstractmethod
-    def run_metric(self, metric_name: str,
-                   input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
-        pass
-
-    @abstractmethod
-    def run_custom_loss(self, custom_loss_name: str,
-                        input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
-        pass
-
-    @abstractmethod
-    def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
-                               ) -> npt.NDArray[np.float32]:
-        pass
-
-    @abstractmethod
-    def get_dataset_setup_response(self, handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
-        pass
-
-    @abstractmethod
-    def get_model_setup_response(self) -> ModelSetup:
-        pass
-
-    @abstractmethod
-    def get_preprocess_sample_ids(
-            self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
-        pass
-
-    @abstractmethod
-    def get_sample_id_type(self) -> Type:
-        pass
-
-    def is_custom_latent_space(self) -> bool:
-        if not self.code_entry_name or not self.code_path:
-            return False
-        custom_layers = self.custom_layers()
-        return any(layer.use_custom_latent_space for layer in custom_layers.values())