code-loader 1.0.64a1__tar.gz → 1.0.64.dev2__tar.gz

{code_loader-1.0.64a1 → code_loader-1.0.64.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.64a1
+Version: 1.0.64.dev2
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

code_loader-1.0.64.dev2/code_loader/code_inegration_processes_manager.py

@@ -0,0 +1,83 @@
+# mypy: ignore-errors
+import traceback
+from dataclasses import dataclass
+
+from typing import List, Tuple, Optional
+
+from multiprocessing import Process, Queue
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from code_loader.leaploader import LeapLoader
+from code_loader.contract.enums import DataStateEnum
+from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
+from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
+
+
+@dataclass
+class SampleSerializableError:
+    state: DataStateEnum
+    index: int
+    leap_script_trace: str
+    exception_as_str: str
+
+
+class CodeIntegrationProcessesManager:
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
+        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=CodeIntegrationProcessesManager._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def _run_and_warm_first_process(self):
+        process = self._create_and_start_process()
+        self.processes = [process]
+
+        # needed in order to make sure the preprocess func runs once in nonparallel
+        self._start_process_inputs([(DataStateEnum.training, 0)])
+        self._get_next_ready_processed_result()
+
+    def _operation_decider(self):
+        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
+            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'metric'
+
+        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
+            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'dataset'
+
+
+
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      samples_to_process: Queue, ready_samples: Queue,
+                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+        while True:
+
+            # decide on sample or metric to process
+            state, idx = samples_to_process.get(block=True)
+            leap_loader._preprocess_result()
+            try:
+                sample = leap_loader.get_sample(state, idx)
+            except Exception as e:
+                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
+                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
+                continue
+
+            ready_samples.put(sample)
+
+    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
+        return self.start_process_inputs(sample_identities)
+

{code_loader-1.0.64a1 → code_loader-1.0.64.dev2}/code_loader/contract/datasetclasses.py

@@ -117,31 +117,46 @@ MetricCallableReturnType = Union[Any, List[List[ConfusionMatrixElement]]]
 
 
 @dataclass
-class CustomLossHandler:
+class CustomLossHandlerData:
     name: str
-    function: CustomCallableInterface
     arg_names: List[str]
 
 
 @dataclass
-class MetricHandler:
+class CustomLossHandler:
+    custom_loss_handler_data: CustomLossHandlerData
+    function: CustomCallableInterface
+
+
+@dataclass
+class MetricHandlerData:
     name: str
-    function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
     arg_names: List[str]
     direction: Optional[MetricDirection] = MetricDirection.Downward
 
 
+@dataclass
+class MetricHandler:
+    metric_handler_data: MetricHandlerData
+    function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
+
+
 @dataclass
 class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
 
 
 @dataclass
-class VisualizerHandler:
+class VisualizerHandlerData:
     name: str
-    function: VisualizerCallableInterface
     type: LeapDataType
     arg_names: List[str]
+
+
+@dataclass
+class VisualizerHandler:
+    visualizer_handler_data: VisualizerHandlerData
+    function: VisualizerCallableInterface
     heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None
 
 

{code_loader-1.0.64a1 → code_loader-1.0.64.dev2}/code_loader/contract/visualizer_classes.py

@@ -1,4 +1,4 @@
-from typing import List, Any, Union, Optional
+from typing import List, Any, Union
 
 import numpy as np
 import numpy.typing as npt
@@ -112,12 +112,11 @@ class LeapText:
 
     Example:
         text_data = ['I', 'ate', 'a', 'banana', '', '', '']
-        heatmap = [0.1, 0.3, 0.2, 0.9, 0.0, 0.0, 0.0]
-        leap_text = LeapText(data=text_data heatmap=heatmap)  # Create LeapText object
+        leap_text = LeapText(data=text_data)  # Create LeapText object
+        LeapText(leap_text)
     """
     data: List[str]
     type: LeapDataType = LeapDataType.Text
-    heatmap: Optional[List[float]] = None
 
     def __post_init__(self) -> None:
         validate_type(self.type, LeapDataType.Text)
@@ -125,15 +124,6 @@ class LeapText:
         for value in self.data:
             validate_type(type(value), str)
 
-        if self.heatmap is not None:
-            validate_type(type(self.heatmap), list)
-            for value in self.heatmap:
-                validate_type(type(value), float)
-            if len(self.heatmap) != len(self.data):
-                raise LeapValidationError(
-                    f"Heatmap length ({len(self.heatmap)}) must match the number of tokens in `data` ({len(self.data)})."
-                )
-
 
 @dataclass
 class LeapHorizontalBar:

code_loader-1.0.64.dev2/code_loader/default_metrics.py

@@ -0,0 +1,81 @@
+from enum import Enum
+from typing import List, Tuple
+import numpy as np
+
+from code_loader.contract.datasetclasses import ConfusionMatrixElement  # type: ignore
+from code_loader.contract.enums import ConfusionMatrixValue, MetricDirection  # type: ignore
+
+
+class Metric(Enum):
+    MeanSquaredError = 'MeanSquaredError'
+    MeanSquaredLogarithmicError = 'MeanSquaredLogarithmicError'
+    MeanAbsoluteError = 'MeanAbsoluteError'
+    MeanAbsolutePercentageError = 'MeanAbsolutePercentageError'
+    Accuracy = 'Accuracy'
+    ConfusionMatrixClassification = 'ConfusionMatrixClassification'
+
+
+def accuracy_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return np.mean((np.round(prediction).astype(np.bool_) == ground_truth.astype(np.bool_)), axis=1)
+
+
+def mean_squared_error_dimension_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return ((ground_truth - prediction) ** 2).mean(axis=1).astype(np.float32)
+
+
+def mean_absolute_error_dimension_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return np.abs(ground_truth - prediction).mean(axis=1).astype(np.float32)
+
+
+def mean_absolute_percentage_error_dimension_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return (np.abs(ground_truth - prediction) / np.abs(ground_truth)).mean(axis=1).astype(np.float32)
+
+
+def mean_squared_logarithmic_error_dimension_reduced(ground_truth: np.array, prediction: np.array) -> np.array:
+    ground_truth, prediction = flatten_non_batch_dims(ground_truth, prediction)
+    return np.mean((np.log(1 + ground_truth) - np.log(1 + prediction)) ** 2, axis=1).astype(np.float32)
+
+
+def flatten_non_batch_dims(ground_truth: np.array, prediction: np.array) -> Tuple[np.array, np.array]:
+    batch_size = ground_truth.shape[0]
+    ground_truth = np.reshape(ground_truth, (batch_size, -1))
+    prediction = np.reshape(prediction, (batch_size, -1))
+    return ground_truth, prediction
+
+
+def confusion_matrix_classification_metric(ground_truth, prediction) -> List[List[ConfusionMatrixElement]]:
+    num_labels = prediction.shape[-1]
+    labels = [str(i) for i in range(num_labels)]
+    if len(labels) == 1:
+        labels = ['0', '1']
+        ground_truth = np.concatenate([1 - ground_truth, ground_truth], axis=1)
+        prediction = np.concatenate([1 - prediction, prediction], axis=1)
+
+    ret = []
+    for batch_i in range(ground_truth.shape[0]):
+        one_hot_vec = list(ground_truth[batch_i])
+        pred_vec = list(prediction[batch_i])
+        confusion_matrix_elements = []
+        for i, label in enumerate(labels):
+            expected_outcome = ConfusionMatrixValue.Positive if int(
+                one_hot_vec[i]) == 1 else ConfusionMatrixValue.Negative
+            cm_element = ConfusionMatrixElement(label, expected_outcome, float(pred_vec[i]))
+            confusion_matrix_elements.append(cm_element)
+        ret.append(confusion_matrix_elements)
+    return ret
+
+
+metrics_names_to_functions_and_direction = {
+    Metric.MeanSquaredError.name: (mean_squared_error_dimension_reduced, MetricDirection.Downward),
+    Metric.MeanSquaredLogarithmicError.name: (
+        mean_squared_logarithmic_error_dimension_reduced, MetricDirection.Downward),
+    Metric.MeanAbsoluteError.name: (mean_absolute_error_dimension_reduced, MetricDirection.Downward),
+    Metric.MeanAbsolutePercentageError.name: (
+        mean_absolute_percentage_error_dimension_reduced, MetricDirection.Downward),
+    Metric.Accuracy.name: (accuracy_reduced, MetricDirection.Upward),
+    Metric.ConfusionMatrixClassification.name: (confusion_matrix_classification_metric, None),
+}

{code_loader-1.0.64a1 → code_loader-1.0.64.dev2}/code_loader/inner_leap_binder/leapbinder.py

@@ -9,10 +9,12 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
-    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, RawInputsForHeatmap
+    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
+    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
+from code_loader.default_metrics import metrics_names_to_functions_and_direction
 from code_loader.utils import to_numpy_return_wrapper, get_shape
 from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
     default_graph_visualizer, \
@@ -36,6 +38,7 @@ class LeapBinder:
         self._visualizer_names: List[str] = list()
         self._encoder_names: List[str] = list()
         self._extend_with_default_visualizers()
+        self._extend_with_default_metrics()
 
         self.batch_size_to_validate: Optional[int] = None
 
@@ -55,6 +58,10 @@ class LeapBinder:
         self.set_visualizer(function=default_text_mask_visualizer, name=DefaultVisualizer.TextMask.value,
                             visualizer_type=LeapDataType.TextMask)
 
+    def _extend_with_default_metrics(self) -> None:
+        for metric_name, (func, direction) in metrics_names_to_functions_and_direction.items():
+            self.add_custom_metric(func, metric_name, direction)
+
     def set_visualizer(self, function: VisualizerCallableInterface,
                        name: str,
                        visualizer_type: LeapDataType,
@@ -127,7 +134,7 @@ class LeapBinder:
                     f'should be {expected_return_type}')
 
         self.setup_container.visualizers.append(
-            VisualizerHandler(name, function, visualizer_type, arg_names, heatmap_visualizer))
+            VisualizerHandler(VisualizerHandlerData(name, visualizer_type, arg_names), function, heatmap_visualizer))
         self._visualizer_names.append(name)
 
     def set_preprocess(self, function: Callable[[], List[PreprocessResponse]]) -> None:
@@ -226,7 +233,7 @@ class LeapBinder:
             leap_binder.add_custom_loss(custom_loss_function, name='custom_loss')
         """
         arg_names = inspect.getfullargspec(function)[0]
-        self.setup_container.custom_loss_handlers.append(CustomLossHandler(name, function, arg_names))
+        self.setup_container.custom_loss_handlers.append(CustomLossHandler(CustomLossHandlerData(name, arg_names), function))
 
     def add_custom_metric(self,
                           function: Union[CustomCallableInterfaceMultiArgs,
@@ -252,7 +259,7 @@ class LeapBinder:
             leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
         """
         arg_names = inspect.getfullargspec(function)[0]
-        self.setup_container.metrics.append(MetricHandler(name, function, arg_names, direction))
+        self.setup_container.metrics.append(MetricHandler(MetricHandlerData(name, arg_names, direction), function))
 
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1) -> None:
         """

{code_loader-1.0.64a1 → code_loader-1.0.64.dev2}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -22,7 +22,7 @@ def tensorleap_custom_metric(name: str, direction: Optional[MetricDirection] = M
             ConfusionMatrixCallableInterfaceMultiArgs]
     ):
         for metric_handler in leap_binder.setup_container.metrics:
-            if metric_handler.name == name:
+            if metric_handler.metric_handler_data.name == name:
                 raise Exception(f'Metric with name {name} already exists. '
                                 f'Please choose another')
 
@@ -94,7 +94,7 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
                                  heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None):
     def decorating_function(user_function: VisualizerCallableInterface):
         for viz_handler in leap_binder.setup_container.visualizers:
-            if viz_handler.name == name:
+            if viz_handler.visualizer_handler_data.name == name:
                 raise Exception(f'Visualizer with name {name} already exists. '
                                 f'Please choose another')
 
@@ -334,7 +334,7 @@ def tensorleap_gt_encoder(name: str):
 def tensorleap_custom_loss(name: str):
     def decorating_function(user_function: CustomCallableInterface):
         for loss_handler in leap_binder.setup_container.custom_loss_handlers:
-            if loss_handler.name == name:
+            if loss_handler.custom_loss_handler_data.name == name:
                 raise Exception(f'Custom loss with name {name} already exists. '
                                 f'Please choose another')
 

{code_loader-1.0.64a1 → code_loader-1.0.64.dev2}/code_loader/leaploader.py

@@ -1,32 +1,33 @@
 # mypy: ignore-errors
 import importlib.util
+import inspect
 import io
 import sys
-import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union, Any, Type
+from typing import Dict, List, Iterable, Union, Any, Type, Optional
 
 import numpy as np
 import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
-    PreprocessResponse, VisualizerHandler, LeapData, CustomLossHandler, \
-    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler
+    PreprocessResponse, VisualizerHandler, LeapData, \
+    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
+    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance
 from code_loader.inner_leap_binder import global_leap_binder
+from code_loader.leaploaderbase import LeapLoaderBase
 from code_loader.utils import get_root_exception_file_and_line_number
 
 
-class LeapLoader:
+class LeapLoader(LeapLoaderBase):
     def __init__(self, code_path: str, code_entry_name: str):
-        self.code_entry_name = code_entry_name
-        self.code_path = code_path
+        super().__init__(code_path, code_entry_name)
 
         self._preprocess_result_cached = None
 
@@ -66,29 +67,56 @@ class LeapLoader:
         spec.loader.exec_module(file)
 
     @lru_cache()
-    def metric_by_name(self) -> Dict[str, MetricHandler]:
+    def metric_by_name(self) -> Dict[str, MetricHandlerData]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
-            metric_handler.name: metric_handler
+            metric_handler.metric_handler_data.name: metric_handler.metric_handler_data
             for metric_handler in setup.metrics
         }
 
     @lru_cache()
-    def visualizer_by_name(self) -> Dict[str, VisualizerHandler]:
+    def _metric_handler_by_name(self) -> Dict[str, MetricHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
-            visualizer_handler.name: visualizer_handler
+            metric_handler.metric_handler_data.name: metric_handler
+            for metric_handler in setup.metrics
+        }
+
+    @lru_cache()
+    def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            visualizer_handler.visualizer_handler_data.name: visualizer_handler.visualizer_handler_data
             for visualizer_handler in setup.visualizers
         }
 
     @lru_cache()
-    def custom_loss_by_name(self) -> Dict[str, CustomLossHandler]:
+    def _visualizer_handler_by_name(self) -> Dict[str, VisualizerHandler]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            visualizer_handler.visualizer_handler_data.name: visualizer_handler
+            for visualizer_handler in setup.visualizers
+        }
+
+    @lru_cache()
+    def custom_loss_by_name(self) -> Dict[str, CustomLossHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            custom_loss_handler.custom_loss_handler_data.name: custom_loss_handler.custom_loss_handler_data
+            for custom_loss_handler in setup.custom_loss_handlers
+        }
+
+    @lru_cache()
+    def _custom_loss_handler_by_name(self) -> Dict[str, CustomLossHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
-            custom_loss_handler.name: custom_loss_handler
+            custom_loss_handler.custom_loss_handler_data.name: custom_loss_handler
             for custom_loss_handler in setup.custom_loss_handlers
         }
 
@@ -200,20 +228,41 @@ class LeapLoader:
         all_dataset_base_handlers.extend(global_leap_binder.setup_container.metadata)
         return all_dataset_base_handlers
 
-    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
-                       ) -> LeapData:
-        return self.visualizer_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
+    def run_metric(self, metric_name: str,
+                   input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
+        self._preprocess_result()
+        return self._metric_handler_by_name()[metric_name].function(**input_tensors_by_arg_name)
+
+    def run_custom_loss(self, custom_loss_name: str,
+                        input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
+        return self._custom_loss_handler_by_name()[custom_loss_name].function(**input_tensors_by_arg_name)
+
+    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
+        # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
+        # global param that visualizer is using)
+        self._preprocess_result()
+
+        return self._visualizer_handler_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
 
     def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
                                ) -> npt.NDArray[np.float32]:
-        heatmap_function = self.visualizer_by_name()[visualizer_name].heatmap_function
+        heatmap_function = self._visualizer_handler_by_name()[visualizer_name].heatmap_function
         if heatmap_function is None:
             assert len(input_tensors_by_arg_name) == 1
             return list(input_tensors_by_arg_name.values())[0]
         return heatmap_function(**input_tensors_by_arg_name)
 
-    @staticmethod
-    def get_dataset_setup_response(handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
+    def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
+        heatmap_function = self._visualizer_handler_by_name()[visualizer_name].heatmap_function
+        if heatmap_function is None:
+            return None
+
+        for arg_name, arg_type in inspect.getfullargspec(heatmap_function).annotations.items():
+            if arg_type == RawInputsForHeatmap:
+                return arg_name
+        return None
+
+    def get_dataset_setup_response(self, handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
         setup = global_leap_binder.setup_container
         assert setup.preprocess is not None
 
@@ -262,10 +311,13 @@ class LeapLoader:
                                                               type=dataset_metadata_type))
 
         visualizers = [
-            VisualizerInstance(visualizer_handler.name, visualizer_handler.type, visualizer_handler.arg_names)
+            VisualizerInstance(
+                visualizer_handler.visualizer_handler_data.name, visualizer_handler.visualizer_handler_data.type,
+                visualizer_handler.visualizer_handler_data.arg_names)
             for visualizer_handler in setup.visualizers]
 
-        custom_losses = [CustomLossInstance(custom_loss.name, custom_loss.arg_names)
+        custom_losses = [CustomLossInstance(custom_loss.custom_loss_handler_data.name,
+                                            custom_loss.custom_loss_handler_data.arg_names)
                          for custom_loss in setup.custom_loss_handlers]
 
         prediction_types = []
@@ -276,15 +328,14 @@ class LeapLoader:
 
         metrics = []
         for metric in setup.metrics:
-            metric_inst = MetricInstance(metric.name, metric.arg_names)
+            metric_inst = MetricInstance(metric.metric_handler_data.name, metric.metric_handler_data.arg_names)
             metrics.append(metric_inst)
 
         return DatasetSetup(preprocess=dataset_preprocess, inputs=inputs, outputs=ground_truths,
                             metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,
                             custom_losses=custom_losses, metrics=metrics)
 
-    @staticmethod
-    def get_model_setup_response() -> ModelSetup:
+    def get_model_setup_response(self) -> ModelSetup:
         setup = global_leap_binder.setup_container
         custom_layer_instances = [
             CustomLayerInstance(custom_layer_handler.name, custom_layer_handler.init_arg_names,
@@ -389,3 +440,4 @@ class LeapLoader:
 
         return id_type
 
+

code_loader-1.0.64.dev2/code_loader/leaploaderbase.py

@@ -0,0 +1,95 @@
+# mypy: ignore-errors
+
+from abc import abstractmethod
+
+from typing import Dict, List, Union, Type, Optional
+
+import numpy as np
+import numpy.typing as npt
+
+from code_loader.contract.datasetclasses import DatasetSample, LeapData, \
+    PredictionTypeHandler, CustomLayerHandler, VisualizerHandlerData, MetricHandlerData, MetricCallableReturnType, \
+    CustomLossHandlerData
+from code_loader.contract.enums import DataStateEnum
+from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
+    DatasetSetup, ModelSetup
+
+
+class LeapLoaderBase:
+    def __init__(self, code_path: str, code_entry_name: str):
+        self.code_entry_name = code_entry_name
+        self.code_path = code_path
+
+    @abstractmethod
+    def metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        pass
+
+    @abstractmethod
+    def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
+        pass
+
+    @abstractmethod
+    def custom_loss_by_name(self) -> Dict[str, CustomLossHandlerData]:
+        pass
+
+    @abstractmethod
+    def custom_layers(self) -> Dict[str, CustomLayerHandler]:
+        pass
+
+    @abstractmethod
+    def prediction_type_by_name(self) -> Dict[str, PredictionTypeHandler]:
+        pass
+
+    @abstractmethod
+    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
+        pass
+
+    @abstractmethod
+    def check_dataset(self) -> DatasetIntegParseResult:
+        pass
+
+    @abstractmethod
+    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
+        pass
+
+    @abstractmethod
+    def run_metric(self, metric_name: str,
+                   input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
+        pass
+
+    @abstractmethod
+    def run_custom_loss(self, custom_loss_name: str,
+                        input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
+        pass
+
+    @abstractmethod
+    def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                               ) -> npt.NDArray[np.float32]:
+        pass
+
+    @abstractmethod
+    def get_dataset_setup_response(self, handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
+        pass
+
+    @abstractmethod
+    def get_model_setup_response(self) -> ModelSetup:
+        pass
+
+    @abstractmethod
+    def get_preprocess_sample_ids(
+            self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
+        pass
+
+    @abstractmethod
+    def get_sample_id_type(self) -> Type:
+        pass
+
+    @abstractmethod
+    def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
+        pass
+
+    def is_custom_latent_space(self) -> bool:
+        if not self.code_entry_name or not self.code_path:
+            return False
+        custom_layers = self.custom_layers()
+        return any(layer.use_custom_latent_space for layer in custom_layers.values())

{code_loader-1.0.64a1 → code_loader-1.0.64.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.64a1"
+version = "1.0.64.dev2"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"