code-loader 1.0.125__tar.gz → 1.0.128__tar.gz

{code_loader-1.0.125 → code_loader-1.0.128}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.125
+Version: 1.0.128
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.125 → code_loader-1.0.128}/code_loader/contract/datasetclasses.py

@@ -40,12 +40,10 @@ class PreprocessResponse:
     sample_id_type: Optional[Union[Type[str], Type[int]]] = None
     sample_ids_to_instance_mappings: Optional[Dict[str, List[str]]] = None # in use only for element instance
     instance_to_sample_ids_mappings: Optional[Dict[str, str]] = None # in use only for element instance
-    instance_ids_to_names: Optional[Dict[str, str]] = None # in use only for element instance
 
     def __post_init__(self) -> None:
         assert self.sample_ids_to_instance_mappings is None, f"Keep sample_ids_to_instance_mappings None when initializing PreprocessResponse"
         assert self.instance_to_sample_ids_mappings is None, f"Keep instance_to_sample_ids_mappings None when initializing PreprocessResponse"
-        assert self.instance_ids_to_names is None, f"Keep instance_ids_to_names None when initializing PreprocessResponse"
 
         if self.length is not None and self.sample_ids is None:
             self.sample_ids = [i for i in range(self.length)]
@@ -73,7 +71,9 @@ class ElementInstance:
     mask: npt.NDArray[np.float32]
 
 SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
-InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse], List[ElementInstance]]
+InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse, int], Optional[ElementInstance]]
+InstanceLengthCallableInterface = Callable[[Union[int, str], PreprocessResponse], int]
+
 
 MetadataSectionCallableInterface = Union[
     Callable[[Union[int, str], PreprocessResponse], int],
@@ -257,6 +257,6 @@ class DatasetSample:
     metadata_is_none: Dict[str, bool]
     index: Union[int, str]
     state: DataStateEnum
-    instance_masks: Optional[Dict[str, List[ElementInstance]]] = None
     custom_latent_space: Optional[npt.NDArray[np.float32]] = None
+    instance_masks: Optional[Dict[str, ElementInstance]] = None
 

{code_loader-1.0.125 → code_loader-1.0.128}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -1,5 +1,6 @@
 # mypy: ignore-errors
 import os
+from functools import lru_cache
 from typing import Optional, Union, Callable, List, Dict
 
 import numpy as np
@@ -8,7 +9,8 @@ import numpy.typing as npt
 from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs, \
     CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, CustomCallableInterface, \
     VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
-    ConfusionMatrixElement, SamplePreprocessResponse, PredictionTypeHandler, InstanceCallableInterface, ElementInstance
+    ConfusionMatrixElement, SamplePreprocessResponse, PredictionTypeHandler, InstanceCallableInterface, ElementInstance, \
+    InstanceLengthCallableInterface
 from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType
 from code_loader import leap_binder
 from code_loader.contract.mapping import NodeMapping, NodeMappingType, NodeConnection
@@ -20,6 +22,7 @@ import inspect
 import functools
 
 _called_from_inside_tl_decorator = 0
+_called_from_inside_tl_integration_test_decorator = False
 
 
 def _add_mapping_connection(user_unique_name, connection_destinations, arg_names, name, node_mapping_type):
@@ -45,17 +48,22 @@ def integration_test():
         leap_binder.integration_test_func = integration_test_function
 
         def inner(*args, **kwargs):
-            ret = integration_test_function(*args, **kwargs)
-
+            global _called_from_inside_tl_integration_test_decorator
             try:
-                os.environ[mapping_runtime_mode_env_var_mame] = 'True'
-                integration_test_function(None, None)
-            except Exception as e:
-                print(f'Error during integration test: Make sure to disable any non tensorleap decorators '
-                      f'functions before pushing a new TL version')
+                _called_from_inside_tl_integration_test_decorator = True
+                ret = integration_test_function(*args, **kwargs)
+
+                try:
+                    os.environ[mapping_runtime_mode_env_var_mame] = 'True'
+                    integration_test_function(None, None)
+                except Exception as e:
+                    print(f'Error during integration test: Make sure to disable any non tensorleap decorators '
+                          f'functions before pushing a new TL version')
+                finally:
+                    if mapping_runtime_mode_env_var_mame in os.environ:
+                        del os.environ[mapping_runtime_mode_env_var_mame]
             finally:
-                if mapping_runtime_mode_env_var_mame in os.environ:
-                    del os.environ[mapping_runtime_mode_env_var_mame]
+                _called_from_inside_tl_integration_test_decorator = False
 
         return inner
 
@@ -70,6 +78,7 @@ def tensorleap_load_model(prediction_types: Optional[List[PredictionTypeHandler]
         class TempMapping:
             pass
 
+        @lru_cache()
         def inner():
             class ModelPlaceholder:
                 def __init__(self):
@@ -529,29 +538,24 @@ def tensorleap_preprocess():
 
 
 def tensorleap_element_instance_preprocess(
-        instance_mask_encoder: Callable[[str, PreprocessResponse], List[ElementInstance]]):
+        instance_length_encoder: InstanceLengthCallableInterface):
     def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
         def user_function_instance() -> List[PreprocessResponse]:
             result = user_function()
             for preprocess_response in result:
                 sample_ids_to_instance_mappings = {}
                 instance_to_sample_ids_mappings = {}
-                instance_ids_to_names = {}
                 all_sample_ids = preprocess_response.sample_ids.copy()
                 for sample_id in preprocess_response.sample_ids:
-                    instances_masks = instance_mask_encoder(sample_id, preprocess_response)
-                    instances_ids = [f'{sample_id}_{instance_id}' for instance_id in range(len(instances_masks))]
+                    instances_length = instance_length_encoder(sample_id, preprocess_response)
+                    instances_ids = [f'{sample_id}_{instance_id}' for instance_id in range(instances_length)]
                     sample_ids_to_instance_mappings[sample_id] = instances_ids
                     instance_to_sample_ids_mappings[sample_id] = sample_id
-                    instance_names = [instance.name for instance in instances_masks]
-                    instance_ids_to_names[sample_id] = 'none'
-                    for instance_id, instance_name in zip(instances_ids, instance_names):
+                    for instance_id in instances_ids:
                         instance_to_sample_ids_mappings[instance_id] = sample_id
-                        instance_ids_to_names[instance_id] = instance_name
                     all_sample_ids.extend(instances_ids)
                 preprocess_response.sample_ids_to_instance_mappings = sample_ids_to_instance_mappings
                 preprocess_response.instance_to_sample_ids_mappings = instance_to_sample_ids_mappings
-                preprocess_response.instance_ids_to_names = instance_ids_to_names
                 preprocess_response.sample_ids = all_sample_ids
             return result
 
@@ -620,7 +624,7 @@ def tensorleap_unlabeled_preprocess():
 
 def tensorleap_instances_masks_encoder(name: str):
     def decorating_function(user_function: InstanceCallableInterface):
-        def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):
+        def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse, instance_id: int):
             assert isinstance(sample_id, str), \
                 (f'tensorleap_instances_masks_encoder validation failed: '
                  f'Argument sample_id should be str. Got {type(sample_id)}.')
@@ -631,18 +635,21 @@ def tensorleap_instances_masks_encoder(name: str):
                 (f'tensorleap_instances_masks_encoder validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
+            assert isinstance(instance_id, int), \
+                (f'tensorleap_instances_masks_encoder validation failed: '
+                 f'Argument instance_id should be int. Got {type(instance_id)}.')
 
         def _validate_result(result):
-            assert isinstance(result, list), \
+            assert isinstance(result, ElementInstance) or (result is None), \
                 (f'tensorleap_instances_masks_encoder validation failed: '
-                 f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
+                 f'Unsupported return type. Should be a ElementInstance or None. Got {type(result)}.')
 
-        def inner_without_validate(sample_id, preprocess_response):
+        def inner_without_validate(sample_id, preprocess_response, instance_id):
             global _called_from_inside_tl_decorator
             _called_from_inside_tl_decorator += 1
 
             try:
-                result = user_function(sample_id, preprocess_response)
+                result = user_function(sample_id, preprocess_response, instance_id)
             finally:
                 _called_from_inside_tl_decorator -= 1
 
@@ -650,6 +657,51 @@ def tensorleap_instances_masks_encoder(name: str):
 
         leap_binder.set_instance_masks(inner_without_validate, name)
 
+        def inner(sample_id, preprocess_response, instance_id):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return None
+
+            _validate_input_args(sample_id, preprocess_response, instance_id)
+
+            result = inner_without_validate(sample_id, preprocess_response, instance_id)
+
+            _validate_result(result)
+            return result
+
+        return inner
+
+    return decorating_function
+
+def tensorleap_instances_length_encoder(name: str):
+    def decorating_function(user_function: InstanceLengthCallableInterface):
+        def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):
+            assert isinstance(sample_id, str), \
+                (f'tensorleap_instances_length_encoder validation failed: '
+                 f'Argument sample_id should be str. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_instances_length_encoder validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
+            assert type(sample_id) == preprocess_response.sample_id_type, \
+                (f'tensorleap_instances_length_encoder validation failed: '
+                 f'Argument sample_id should be as the same type as defined in the preprocess response '
+                 f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
+
+        def _validate_result(result):
+            assert isinstance(result, int), \
+                (f'tensorleap_instances_length_encoder validation failed: '
+                 f'Unsupported return type. Should be a int. Got {type(result)}.')
+
+        def inner_without_validate(sample_id, preprocess_response):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(sample_id, preprocess_response)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
         def inner(sample_id, preprocess_response):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
                 return None
@@ -665,7 +717,6 @@ def tensorleap_instances_masks_encoder(name: str):
 
     return decorating_function
 
-
 def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
     def decorating_function(user_function: SectionCallableInterface):
         for input_handler in leap_binder.setup_container.inputs:
@@ -719,7 +770,7 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
 
             _validate_result(result)
 
-            if _called_from_inside_tl_decorator == 0:
+            if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
                 result = np.expand_dims(result, axis=0)
 
             return result
@@ -803,7 +854,7 @@ def tensorleap_gt_encoder(name: str):
 
             _validate_result(result)
 
-            if _called_from_inside_tl_decorator == 0:
+            if _called_from_inside_tl_decorator == 0 and _called_from_inside_tl_integration_test_decorator:
                 result = np.expand_dims(result, axis=0)
 
             return result

{code_loader-1.0.125 → code_loader-1.0.128}/code_loader/leaploader.py

@@ -157,7 +157,7 @@ class LeapLoader(LeapLoaderBase):
             for prediction_type in setup.prediction_types
         }
 
-    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
+    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: int = None) -> DatasetSample:
         self.exec_script()
         preprocess_result = self._preprocess_result()
         if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
@@ -170,30 +170,15 @@ class LeapLoader(LeapLoaderBase):
             custom_latent_space = global_leap_binder.setup_container.custom_latent_space.function(sample_id,
                                                                                                   preprocess_result[
                                                                                                       state])
-
+        instance_mask = self._get_instances_masks(state, sample_id, instance_id)
         sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
                                gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
                                metadata=metadata,
                                metadata_is_none=metadata_is_none,
                                index=sample_id,
                                state=state,
-                               custom_latent_space=custom_latent_space)
-        return sample
-
-    def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
-        self.exec_script()
-        preprocess_result = self._preprocess_result()
-        if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
-            self._preprocess_result(update_unlabeled_preprocess=True)
-
-        metadata, metadata_is_none = self._get_metadata(state, sample_id)
-        sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
-                               gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
-                               metadata=metadata,
-                               metadata_is_none=metadata_is_none,
-                               index=sample_id,
-                               state=state,
-                               instance_masks=self._get_instances_masks(state, sample_id))
+                               custom_latent_space=custom_latent_space,
+                               instance_masks=instance_mask)
         return sample
 
     def check_dataset(self) -> DatasetIntegParseResult:
@@ -484,13 +469,14 @@ class LeapLoader(LeapLoaderBase):
     def _get_inputs(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, sample_id)
 
-    def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[
-        str, List[ElementInstance]]:
+    def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: int) -> Optional[Dict[str, ElementInstance]]:
+        if instance_id is None:
+            return None
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
         result_agg = {}
         for handler in global_leap_binder.setup_container.instance_masks:
-            handler_result = handler.function(sample_id, preprocess_state)
+            handler_result = handler.function(sample_id, preprocess_state, instance_id)
             handler_name = handler.name
             result_agg[handler_name] = handler_result
         return result_agg
@@ -570,7 +556,7 @@ class LeapLoader(LeapLoaderBase):
         self.exec_script()
         return global_leap_binder.setup_container.custom_latent_space is not None
 
-    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str], Dict[str, str]]:
+    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
         """
         This Method get the data state and returns two dictionaries that holds the mapping of the sample ids to their
          instances and the other way around and the sample ids array.
@@ -583,4 +569,4 @@ class LeapLoader(LeapLoaderBase):
         """
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
-        return preprocess_state.sample_ids_to_instance_mappings, preprocess_state.instance_to_sample_ids_mappings, preprocess_state.instance_ids_to_names
+        return preprocess_state.sample_ids_to_instance_mappings, preprocess_state.instance_to_sample_ids_mappings

{code_loader-1.0.125 → code_loader-1.0.128}/code_loader/leaploaderbase.py

@@ -61,15 +61,11 @@ class LeapLoaderBase:
         pass
 
     @abstractmethod
-    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
+    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: int = None) -> DatasetSample:
         pass
 
     @abstractmethod
-    def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
-        pass
-
-    @abstractmethod
-    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str], Dict[str, str]]:
+    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
         pass
 
     @abstractmethod

{code_loader-1.0.125 → code_loader-1.0.128}/code_loader/plot_functions/plot_functions.py

@@ -29,7 +29,6 @@ def run_only_on_non_mapping_mode():
     def decorator(func):
         def wrapper(*args, **kwargs):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
-                print(f"Skipping {func.__name__} in mapping mode.")
                 return
             return func(*args, **kwargs)
         return wrapper

{code_loader-1.0.125 → code_loader-1.0.128}/code_loader/utils.py

@@ -18,12 +18,12 @@ def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> Secti
 
     return numpy_encoder_function
 
-def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -> Callable[
-    [Union[int, str], PreprocessResponse], List[ElementInstance]]:
-    def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse) -> List[ElementInstance]:
-        result = encoder_function(idx, samples)
-        for res in result:
-            res.mask = np.array(res.mask)
+def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -> InstanceCallableInterface:
+    def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse, element_idx: int) -> Union[ElementInstance, None]:
+        result = encoder_function(idx, samples, element_idx)
+        if result is None:
+            return None
+        result.mask = np.array(result.mask)
         return result
     return numpy_encoder_function
 

{code_loader-1.0.125 → code_loader-1.0.128}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.125"
+version = "1.0.128"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"