code-loader 1.0.94.dev2__py3-none-any.whl → 1.0.99.dev1__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -38,18 +38,11 @@ class PreprocessResponse:
     sample_ids: Optional[Union[List[str], List[int]]] = None
     state: Optional[DataStateType] = None
     sample_id_type: Optional[Union[Type[str], Type[int]]] = None
-    sample_ids_to_instance_mappings: Optional[Dict[str, List[str]]] = None # in use only for element instance
-    instance_to_sample_ids_mappings: Optional[Dict[str, str]] = None # in use only for element instance
-    instance_id_to_instance_name: Optional[Dict[str, str]] = None # in use only for element instance
 
     def __post_init__(self) -> None:
         def is_valid_string(s: str) -> bool:
             return bool(re.match(r'^[A-Za-z0-9_]+$', s))
 
-        assert self.sample_ids_to_instance_mappings is None, f"Keep sample_ids_to_instance_mappings None when initializing PreprocessResponse"
-        assert self.instance_to_sample_ids_mappings is None, f"Keep instance_to_sample_ids_mappings None when initializing PreprocessResponse"
-        assert self.instance_id_to_instance_name is None, f"Keep instance_id_to_instance_name None when initializing PreprocessResponse"
-
         if self.length is not None and self.sample_ids is None:
             self.sample_ids = [i for i in range(self.length)]
             self.sample_id_type = int
@@ -72,13 +65,8 @@ class PreprocessResponse:
         assert self.sample_ids is not None
         return len(self.sample_ids)
 
-@dataclass
-class ElementInstance:
-    name: str
-    mask: npt.NDArray[np.float32]
 
 SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
-InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse], List[ElementInstance]]
 
 MetadataSectionCallableInterface = Union[
     Callable[[Union[int, str], PreprocessResponse], int],
@@ -200,10 +188,6 @@ class InputHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
     channel_dim: Optional[int] = -1
 
-@dataclass
-class ElementInstanceMasksHandler:
-    name: str
-    function: InstanceCallableInterface
 
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):
@@ -221,7 +205,7 @@ class MetadataHandler:
 class PredictionTypeHandler:
     name: str
     labels: List[str]
-    channel_dim: int
+    channel_dim: int = -1
 
 
 @dataclass
@@ -239,7 +223,6 @@ class DatasetIntegrationSetup:
     unlabeled_data_preprocess: Optional[UnlabeledDataPreprocessHandler] = None
     visualizers: List[VisualizerHandler] = field(default_factory=list)
     inputs: List[InputHandler] = field(default_factory=list)
-    instance_masks: List[ElementInstanceMasksHandler] = field(default_factory=list)
     ground_truths: List[GroundTruthHandler] = field(default_factory=list)
     metadata: List[MetadataHandler] = field(default_factory=list)
     prediction_types: List[PredictionTypeHandler] = field(default_factory=list)
@@ -256,5 +239,3 @@ class DatasetSample:
     metadata_is_none: Dict[str, bool]
     index: Union[int, str]
     state: DataStateEnum
-    instance_masks: Optional[Dict[str, List[ElementInstance]]] = None
-

code_loader/contract/visualizer_classes.py

@@ -121,23 +121,19 @@ class LeapGraph:
         x_label = 'Frequency [Seconds]'
         y_label = 'Amplitude [Voltage]'
         x_range = (0.1, 3.0)
-        legend = ['experiment1', 'experiment2', 'experiment3']
-        leap_graph = LeapGraph(data=graph_data, x_label=x_label, y_label=y_label, x_range=x_range, legend=legend)
+        leap_graph = LeapGraph(data=graph_data, x_label=x_label, y_label=y_label, x_range=x_range)
     """
     data: npt.NDArray[np.float32]
     type: LeapDataType = LeapDataType.Graph
     x_label: Optional[str] = None
     y_label: Optional[str] = None
     x_range: Optional[Tuple[float,float]] = None
-    legend: Optional[List[str]] = None
 
     def __post_init__(self) -> None:
         validate_type(self.type, LeapDataType.Graph)
         validate_type(type(self.data), np.ndarray)
         validate_type(self.data.dtype, np.float32)
-        validate_type(len(self.data.shape), 2, f'Graph must be of shape 2')
-        if self.legend:
-            validate_type(self.data.shape[1], len(self.legend), 'Number of labels supplied should equal the number of graphs')
+        validate_type(len(self.data.shape), 2, 'Graph must be of shape 2')
         validate_type(type(self.x_label), [str, type(None)], 'x_label must be a string or None')
         validate_type(type(self.y_label), [str, type(None)], 'y_label must be a string or None')
         validate_type(type(self.x_range), [tuple, type(None)], 'x_range must be a tuple or None')

code_loader/inner_leap_binder/leapbinder.py

@@ -10,21 +10,23 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
-    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface
+    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
 from code_loader.default_losses import loss_name_to_function
 from code_loader.default_metrics import metrics_names_to_functions_and_direction
-from code_loader.utils import to_numpy_return_wrapper, get_shape, to_numpy_return_masks_wrapper
+from code_loader.utils import to_numpy_return_wrapper, get_shape
 from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
     default_graph_visualizer, \
     default_image_visualizer, default_horizontal_bar_visualizer, default_word_visualizer, \
     default_image_mask_visualizer, default_text_mask_visualizer, default_raw_data_visualizer, default_video_visualizer
 
 
+mapping_runtime_mode_env_var_mame = '__MAPPING_RUNTIME_MODE__'
+
+
 class LeapBinder:
     """
     Interface to the Tensorleap platform. Provides methods to set up preprocessing,
@@ -235,22 +237,6 @@ class LeapBinder:
 
         self._encoder_names.append(name)
 
-
-    def set_instance_masks(self, function: InstanceCallableInterface, name: str) -> None:
-        """
-        Set the instance mask handler function.
-
-        Args:
-        function (SectionCallableInterface): The input handler function.
-        name (str): The name of the instance mask section.
-
-        Example:
-            leap_binder.set_input(input_encoder, name='input_encoder')
-        """
-        function = to_numpy_return_masks_wrapper(function)
-        self.setup_container.instance_masks.append(ElementInstanceMasksHandler(name, function))
-
-
     def add_custom_loss(self, function: CustomCallableInterface, name: str) -> None:
         """
         Add a custom loss function to the setup.

code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1,5 +1,5 @@
 # mypy: ignore-errors
-
+import os
 from typing import Optional, Union, Callable, List, Dict
 
 import numpy as np
@@ -8,22 +8,83 @@ import numpy.typing as npt
 from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs, \
     CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, CustomCallableInterface, \
     VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
-    ConfusionMatrixElement, SamplePreprocessResponse, InstanceCallableInterface, ElementInstance
+    ConfusionMatrixElement, SamplePreprocessResponse, PredictionTypeHandler
 from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType
 from code_loader import leap_binder
 from code_loader.contract.mapping import NodeMapping, NodeMappingType, NodeConnection
 from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, LeapTextMask, LeapText, LeapGraph, \
     LeapHorizontalBar, LeapImageWithBBox, LeapImageWithHeatmap
+from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame
+
+
+def _add_mapping_connection(user_unique_name, connection_destinations, arg_names, name, node_mapping_type):
+    main_node_mapping = NodeMapping(name, node_mapping_type, user_unique_name, arg_names=arg_names)
+    node_inputs = {}
+    for arg_name, destination in zip(arg_names, connection_destinations):
+        node_inputs[arg_name] = destination.node_mapping
+
+    leap_binder.mapping_connections.append(NodeConnection(main_node_mapping, node_inputs))
 
 
 def _add_mapping_connections(connects_to, arg_names, node_mapping_type, name):
     for user_unique_name, connection_destinations in connects_to.items():
-        main_node_mapping = NodeMapping(name, node_mapping_type, user_unique_name, arg_names=arg_names)
-        node_inputs = {}
-        for arg_name, destination in zip(arg_names, connection_destinations):
-            node_inputs[arg_name] = destination.node_mapping
+        _add_mapping_connection(user_unique_name, connection_destinations, arg_names, name, node_mapping_type)
+
+
+
+
+def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]] = None):
+    for i, prediction_type in enumerate(prediction_types):
+        leap_binder.add_prediction(prediction_type.name, prediction_type.labels, prediction_type.channel_dim, i)
+
+    def decorating_function(load_model_func):
+        class TempMapping:
+            pass
+
+        def mapping_inner():
+            class ModelOutputPlaceholder:
+                def __init__(self):
+                    self.node_mapping = NodeMapping('', NodeMappingType.Prediction0)
+
+                def __getitem__(self, key):
+                    assert isinstance(key, int), \
+                        f'Expected key to be an int, got {type(key)} instead.'
+
+                    ret = TempMapping()
+                    ret.node_mapping = NodeMapping('', NodeMappingType(f'Prediction{str(key)}'))
+                    return ret
+
+            class ModelPlaceholder:
+                #keras interface
+                def __call__(self, arg):
+                    if isinstance(arg, list):
+                        for i, elem in enumerate(arg):
+                            elem.node_mapping.type = NodeMappingType[f'Input{str(i)}']
+                    else:
+                        arg.node_mapping.type = NodeMappingType.Input0
+
+                    return ModelOutputPlaceholder()
+
+                # onnx runtime interface
+                def run(self, output_names, input_dict):
+                    assert output_names is None
+                    assert isinstance(input_dict, dict), \
+                        f'Expected input_dict to be a dict, got {type(input_dict)} instead.'
+                    for i, elem in enumerate(input_dict.values()):
+                        elem.node_mapping.type = NodeMappingType[f'Input{str(i)}']
+
+                    return ModelOutputPlaceholder()
+
+            return ModelPlaceholder()
+
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return load_model_func
+
+    return decorating_function
 
-        leap_binder.mapping_connections.append(NodeConnection(main_node_mapping, node_inputs))
 
 
 def tensorleap_custom_metric(name: str,
@@ -31,8 +92,8 @@ def tensorleap_custom_metric(name: str,
                              compute_insights: Optional[Union[bool, Dict[str, bool]]] = None,
                              connects_to=None):
     def decorating_function(user_function: Union[CustomCallableInterfaceMultiArgs,
-    CustomMultipleReturnCallableInterfaceMultiArgs,
-    ConfusionMatrixCallableInterfaceMultiArgs]):
+            CustomMultipleReturnCallableInterfaceMultiArgs,
+            ConfusionMatrixCallableInterfaceMultiArgs]):
         for metric_handler in leap_binder.setup_container.metrics:
             if metric_handler.metric_handler_data.name == name:
                 raise Exception(f'Metric with name {name} already exists. '
@@ -119,14 +180,31 @@ def tensorleap_custom_metric(name: str,
                         (f'tensorleap_custom_metric validation failed: '
                          f'compute_insights should be boolean. Got {type(compute_insights)}.')
 
-
         def inner(*args, **kwargs):
             _validate_input_args(*args, **kwargs)
             result = user_function(*args, **kwargs)
             _validate_result(result)
             return result
 
-        return inner
+        def mapping_inner(*args, **kwargs):
+            user_unique_name = mapping_inner.name
+            if 'user_unique_name' in kwargs:
+                user_unique_name = kwargs['user_unique_name']
+
+            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
+            ordered_connections = list(args) + ordered_connections
+            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
+                                    mapping_inner.name,  NodeMappingType.Metric)
+
+            return None
+
+        mapping_inner.arg_names = leap_binder.setup_container.metrics[-1].metric_handler_data.arg_names
+        mapping_inner.name = name
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 
@@ -186,7 +264,25 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
             _validate_result(result)
             return result
 
-        return inner
+        def mapping_inner(*args, **kwargs):
+            user_unique_name = mapping_inner.name
+            if 'user_unique_name' in kwargs:
+                user_unique_name = kwargs['user_unique_name']
+
+            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
+            ordered_connections = list(args) + ordered_connections
+            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
+                                    mapping_inner.name, NodeMappingType.Visualizer)
+
+            return None
+
+        mapping_inner.arg_names = leap_binder.setup_container.visualizers[-1].visualizer_handler_data.arg_names
+        mapping_inner.name = name
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 
@@ -270,66 +366,6 @@ def tensorleap_preprocess():
 
     return decorating_function
 
-def tensorleap_element_instance_preprocess(instance_mask_encoder: Callable[[int, PreprocessResponse], List[ElementInstance]]):
-    def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
-        def user_function_instance() -> List[PreprocessResponse]:
-            result = user_function()
-            for preprocess_response in result:
-                sample_ids_to_instance_mappings = {}
-                instance_to_sample_ids_mappings = {}
-                instance_id_to_instance_name = {}
-                all_sample_ids = preprocess_response.sample_ids.copy()
-                for sample_id in preprocess_response.sample_ids:
-                    instances_masks = instance_mask_encoder(sample_id, preprocess_response)
-                    instance_names = [instance.name for instance in instances_masks]
-                    instances_ids = [f'{sample_id}_{instance_id}' for instance_id in range(len(instances_masks))]
-                    sample_ids_to_instance_mappings[sample_id] = instances_ids
-                    instance_to_sample_ids_mappings[sample_id] = sample_id
-                    instance_id_to_instance_name[sample_id] = None
-                    for instance_id, instance_name in zip(instances_ids, instance_names):
-                        instance_to_sample_ids_mappings[instance_id] = sample_id
-                        instance_id_to_instance_name[instance_id] = instance_name
-                    all_sample_ids.extend(instances_ids)
-                preprocess_response.sample_ids_to_instance_mappings = sample_ids_to_instance_mappings
-                preprocess_response.instance_to_sample_ids_mappings = instance_to_sample_ids_mappings
-                preprocess_response.instance_id_to_instance_name = instance_id_to_instance_name
-                preprocess_response.sample_ids = all_sample_ids
-            return result
-
-        def metadata_is_instance(idx: str, preprocess: PreprocessResponse) -> Dict[str, str]:
-            return {'is_instance': '0',
-                    'orig_sample_id': preprocess.instance_to_sample_ids_mappings[idx],
-                    'instance_name': preprocess.instance_id_to_instance_name[idx]}
-        leap_binder.set_preprocess(user_function_instance)
-        leap_binder.set_metadata(metadata_is_instance, "metadata_is_instance")
-
-        def _validate_input_args(*args, **kwargs):
-            assert len(args) == 0 and len(kwargs) == 0, \
-                (f'tensorleap_element_instance_preprocess validation failed: '
-                 f'The function should not take any arguments. Got {args} and {kwargs}.')
-
-        def _validate_result(result):
-            assert isinstance(result, list), \
-                (f'tensorleap_element_instance_preprocess validation failed: '
-                 f'The return type should be a list. Got {type(result)}.')
-            for i, response in enumerate(result):
-                assert isinstance(response, PreprocessResponse), \
-                    (f'tensorleap_element_instance_preprocess validation failed: '
-                     f'Element #{i} in the return list should be a PreprocessResponse. Got {type(response)}.')
-            assert len(set(result)) == len(result), \
-                (f'tensorleap_element_instance_preprocess validation failed: '
-                 f'The return list should not contain duplicate PreprocessResponse objects.')
-
-        def inner(*args, **kwargs):
-            _validate_input_args(*args, **kwargs)
-            result = user_function_instance()
-            _validate_result(result)
-            return result
-
-        return inner
-
-    return decorating_function
-
 
 def tensorleap_unlabeled_preprocess():
     def decorating_function(user_function: Callable[[], PreprocessResponse]):
@@ -356,38 +392,6 @@ def tensorleap_unlabeled_preprocess():
     return decorating_function
 
 
-def tensorleap_instances_masks_encoder(name: str):
-    def decorating_function(user_function: InstanceCallableInterface):
-        leap_binder.set_instance_masks(user_function, name)
-
-        def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):
-            assert isinstance(sample_id, str), \
-                (f'tensorleap_instances_masks_encoder validation failed: '
-                 f'Argument sample_id should be str. Got {type(sample_id)}.')
-            assert isinstance(preprocess_response, PreprocessResponse), \
-                (f'tensorleap_instances_masks_encoder validation failed: '
-                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
-            assert type(sample_id) == preprocess_response.sample_id_type, \
-                (f'tensorleap_instances_masks_encoder validation failed: '
-                 f'Argument sample_id should be as the same type as defined in the preprocess response '
-                 f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
-
-        def _validate_result(result):
-            assert isinstance(result, list), \
-                (f'tensorleap_instances_masks_encoder validation failed: '
-                 f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
-
-        def inner(sample_id, preprocess_response):
-            _validate_input_args(sample_id, preprocess_response)
-            result = user_function(sample_id, preprocess_response)
-            _validate_result(result)
-            return result
-
-        return inner
-
-    return decorating_function
-
-
 def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
     def decorating_function(user_function: SectionCallableInterface):
         for input_handler in leap_binder.setup_container.inputs:
@@ -432,7 +436,21 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
             node_mapping_type = NodeMappingType(f'Input{str(model_input_index)}')
         inner.node_mapping = NodeMapping(name, node_mapping_type)
 
-        return inner
+
+        def mapping_inner(*args, **kwargs):
+            class TempMapping:
+                pass
+            ret = TempMapping()
+            ret.node_mapping = mapping_inner.node_mapping
+
+            return ret
+
+        mapping_inner.node_mapping = NodeMapping(name, node_mapping_type)
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 
@@ -474,9 +492,20 @@ def tensorleap_gt_encoder(name: str):
 
         inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
-        return inner
+        def mapping_inner(*args, **kwargs):
+            class TempMapping:
+                pass
+            ret = TempMapping()
+            ret.node_mapping = mapping_inner.node_mapping
+
+            return ret
 
+        mapping_inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
 
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 
@@ -532,7 +561,25 @@ def tensorleap_custom_loss(name: str, connects_to=None):
             _validate_result(result)
             return result
 
-        return inner
+        def mapping_inner(*args, **kwargs):
+            user_unique_name = mapping_inner.name
+            if 'user_unique_name' in kwargs:
+                user_unique_name = kwargs['user_unique_name']
+
+            ordered_connections = [kwargs[n] for n in mapping_inner.arg_names if n in kwargs]
+            ordered_connections = list(args) + ordered_connections
+            _add_mapping_connection(user_unique_name, ordered_connections, mapping_inner.arg_names,
+                                    mapping_inner.name, NodeMappingType.CustomLoss)
+
+            return None
+
+        mapping_inner.arg_names = leap_binder.setup_container.custom_loss_handlers[-1].custom_loss_handler_data.arg_names
+        mapping_inner.name = name
+
+        if os.environ[mapping_runtime_mode_env_var_mame]:
+            return mapping_inner
+        else:
+            return inner
 
     return decorating_function
 

code_loader/leaploader.py

@@ -2,6 +2,7 @@
 import importlib.util
 import inspect
 import io
+import os
 import sys
 from contextlib import redirect_stdout
 from functools import lru_cache
@@ -14,8 +15,7 @@ import numpy.typing as npt
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
-    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
-    ElementInstance
+    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -23,6 +23,7 @@ from code_loader.contract.responsedataclasses import DatasetIntegParseResult, Da
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance, \
     EngineFileContract
 from code_loader.inner_leap_binder import global_leap_binder
+from code_loader.inner_leap_binder.leapbinder import mapping_runtime_mode_env_var_mame
 from code_loader.leaploaderbase import LeapLoaderBase
 from code_loader.utils import get_root_exception_file_and_line_number
 
@@ -151,22 +152,6 @@ class LeapLoader(LeapLoaderBase):
                                state=state)
         return sample
 
-    def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
-        self.exec_script()
-        preprocess_result = self._preprocess_result()
-        if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
-            self._preprocess_result(update_unlabeled_preprocess=True)
-
-        metadata, metadata_is_none = self._get_metadata(state, sample_id)
-        sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
-                               gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
-                               metadata=metadata,
-                               metadata_is_none=metadata_is_none,
-                               index=sample_id,
-                               state=state,
-                               instance_masks=self._get_instances_masks(state, sample_id))
-        return sample
-
     def check_dataset(self) -> DatasetIntegParseResult:
         test_payloads: List[DatasetTestResultPayload] = []
         setup_response = None
@@ -174,7 +159,23 @@ class LeapLoader(LeapLoaderBase):
         stdout_steam = io.StringIO()
         with redirect_stdout(stdout_steam):
             try:
+                # generate mapping
+                os.environ[mapping_runtime_mode_env_var_mame] = 'TRUE'
+                self.exec_script()
+                mapping_connections = global_leap_binder.mapping_connections
+                del os.environ[mapping_runtime_mode_env_var_mame]
+
+                # init global leap binder
+                global_leap_binder.__init__()
+                self.exec_script.cache_clear()
+
+                # run parse without mapping
                 self.exec_script()
+
+                # set mapping connections
+                if not global_leap_binder.mapping_connections:
+                    global_leap_binder.mapping_connections = mapping_connections
+
                 preprocess_test_payload = self._check_preprocess()
                 test_payloads.append(preprocess_test_payload)
                 handlers_test_payloads = self._check_handlers()
@@ -190,6 +191,8 @@ class LeapLoader(LeapLoaderBase):
                 general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                 is_valid = False
 
+
+
         print_log = stdout_steam.getvalue()
         is_valid_for_model = bool(global_leap_binder.setup_container.custom_layers)
         model_setup = self.get_model_setup_response()
@@ -454,16 +457,6 @@ class LeapLoader(LeapLoaderBase):
     def _get_inputs(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, sample_id)
 
-    def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, List[ElementInstance]]:
-        preprocess_result = self._preprocess_result()
-        preprocess_state = preprocess_result[state]
-        result_agg = {}
-        for handler in global_leap_binder.setup_container.instance_masks:
-            handler_result = handler.function(sample_id, preprocess_state)
-            handler_name = handler.name
-            result_agg[handler_name] = handler_result
-        return result_agg
-
     def _get_gt(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.ground_truths, state, sample_id)
 
@@ -532,18 +525,3 @@ class LeapLoader(LeapLoaderBase):
                 raise Exception("Different id types in preprocess results")
 
         return id_type
-
-    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[Union[int, str], List[Union[int, str]]], Dict[Union[int, str], Union[int, str]], List[Union[int, str]]]:
-        """
-        This Method get the data state and returns two dictionaries that holds the mapping of the sample ids to their
-         instances and the other way around and the sample ids array.
-        Args:
-            state: DataStateEnum state
-        Returns:
-            sample_ids_to_instance_mappings: sample id to instance mappings
-            instance_to_sample_ids_mappings: instance to sample ids mappings
-            sample_ids: sample ids array
-        """
-        preprocess_result = self._preprocess_result()
-        preprocess_state = preprocess_result[state]
-        return preprocess_state.sample_ids_to_instance_mappings, preprocess_state.instance_to_sample_ids_mappings, preprocess_state.sample_ids

code_loader/leaploaderbase.py

@@ -2,7 +2,7 @@
 
 from abc import abstractmethod
 
-from typing import Dict, List, Union, Type, Optional, Tuple
+from typing import Dict, List, Union, Type, Optional
 
 import numpy as np
 import numpy.typing as npt
@@ -64,14 +64,6 @@ class LeapLoaderBase:
     def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
         pass
 
-    @abstractmethod
-    def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
-        pass
-
-    @abstractmethod
-    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[Union[int, str], List[Union[int, str]]], Dict[Union[int, str], Union[int, str]], List[Union[int, str]]]:
-        pass
-
     @abstractmethod
     def check_dataset(self) -> DatasetIntegParseResult:
         pass

code_loader/utils.py

@@ -1,13 +1,12 @@
 import sys
 from pathlib import Path
 from types import TracebackType
-from typing import List, Union, Tuple, Any, Callable
+from typing import List, Union, Tuple, Any
 import traceback
 import numpy as np
 import numpy.typing as npt
 
-from code_loader.contract.datasetclasses import SectionCallableInterface, PreprocessResponse, \
-    InstanceCallableInterface, ElementInstance
+from code_loader.contract.datasetclasses import SectionCallableInterface, PreprocessResponse
 
 
 def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> SectionCallableInterface:
@@ -18,15 +17,6 @@ def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> Secti
 
     return numpy_encoder_function
 
-def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -> Callable[
-    [Union[int, str], PreprocessResponse], List[ElementInstance]]:
-    def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse) -> List[ElementInstance]:
-        result = encoder_function(idx, samples)
-        for res in result:
-            res.mask = np.array(res.mask)
-        return result
-    return numpy_encoder_function
-
 
 def get_root_traceback(exc_tb: TracebackType) -> TracebackType:
     return_traceback = exc_tb

{code_loader-1.0.94.dev2.dist-info → code_loader-1.0.99.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.94.dev2
+Version: 1.0.99.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.94.dev2.dist-info → code_loader-1.0.99.dev1.dist-info}/RECORD

@@ -1,12 +1,12 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=PT1dMA0FxpzJ75rpc79d_oxn3zJmrdOihKTC46ZEZvI,9139
+code_loader/contract/datasetclasses.py,sha256=sNBHyfdg2uS3rL65kJVZjjNVbSXy0dTQMganldQjGNw,7969
 code_loader/contract/enums.py,sha256=GEFkvUMXnCNt-GOoz7NJ9ecQZ2PPDettJNOsxsiM0wk,1622
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/mapping.py,sha256=e11h_sprwOyE32PcqgRq9JvyahQrPzwqgkhmbQLKLQY,1165
 code_loader/contract/responsedataclasses.py,sha256=6-5DJkYBdXb3UB1eNidTTPPBIYxMjEoMdYDkp9VhH8o,4223
-code_loader/contract/visualizer_classes.py,sha256=Wz9eItmoRaKEHa3p0aW0Ypxx4_xUmaZyLBznnTuxwi0,15425
+code_loader/contract/visualizer_classes.py,sha256=_nlukRfW8QeQaQG7G5HfAUSeCuoqslB4xJS2AGI_Nz8,15156
 code_loader/default_losses.py,sha256=NoOQym1106bDN5dcIk56Elr7ZG5quUHArqfP5-Nyxyo,1139
 code_loader/default_metrics.py,sha256=v16Mrt2Ze1tXPgfKywGVdRSrkaK4CKLNQztN1UdVqIY,5010
 code_loader/experiment_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,14 +20,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=wmCOj_YKbRXqLL1k5Tw_FrcZgfmgnVQcjs2ok6wdlww,32362
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=KNbfm8o7F-3ljys5fbVK8NP6mkPSD_itFceGFTWk4kw,29657
-code_loader/leaploader.py,sha256=HPDZb10HPYh18_HjoIYT8ipB5pmVvL5tEI_KFKmHS7g,28866
-code_loader/leaploaderbase.py,sha256=tpMVEd97675b_var4hvesjN7EgQzoCbPEayNBut6AvI,4551
-code_loader/utils.py,sha256=_j8b60pimoNAvWMRj7hEkkT6C76qES6cZoBFHpXHMxA,2698
+code_loader/inner_leap_binder/leapbinder.py,sha256=FaX0kT-dEQJx4tS3LBfDxjf7Bi1LihkeEfLDHbyEbnM,31778
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=xvQw6vVYO8acINLzzkePWetsrWvPBiDp6oge8t3GnC0,30044
+code_loader/leaploader.py,sha256=LN7ngforPp7zGd6_1tONpi4AvOry-wTg7agH0D-gn1c,27233
+code_loader/leaploaderbase.py,sha256=VH0vddRmkqLtcDlYPCO7hfz1_VbKo43lUdHDAbd4iJc,4198
+code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=669lBpLISLO6my5Qcgn1FLDDeZgHumPf252m4KHY4YM,2555
-code_loader-1.0.94.dev2.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.94.dev2.dist-info/METADATA,sha256=5rh-Eu9KRqvossEgh3kk7iSDstW5JYfVnuo6xAhFFCY,854
-code_loader-1.0.94.dev2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.94.dev2.dist-info/RECORD,,
+code_loader-1.0.99.dev1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.99.dev1.dist-info/METADATA,sha256=LfhWldl2Byaa2GpWaY2P_V-2xPz8c_tpAMeK6kwrDBI,854
+code_loader-1.0.99.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.99.dev1.dist-info/RECORD,,