code-loader 1.0.119rc2__py3-none-any.whl → 1.0.121__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -40,12 +40,10 @@ class PreprocessResponse:
     sample_id_type: Optional[Union[Type[str], Type[int]]] = None
     sample_ids_to_instance_mappings: Optional[Dict[str, List[str]]] = None # in use only for element instance
     instance_to_sample_ids_mappings: Optional[Dict[str, str]] = None # in use only for element instance
-    instance_ids_to_names: Optional[Dict[str, str]] = None # in use only for element instance
 
     def __post_init__(self) -> None:
         assert self.sample_ids_to_instance_mappings is None, f"Keep sample_ids_to_instance_mappings None when initializing PreprocessResponse"
         assert self.instance_to_sample_ids_mappings is None, f"Keep instance_to_sample_ids_mappings None when initializing PreprocessResponse"
-        assert self.instance_ids_to_names is None, f"Keep instance_ids_to_names None when initializing PreprocessResponse"
 
         if self.length is not None and self.sample_ids is None:
             self.sample_ids = [i for i in range(self.length)]
@@ -73,7 +71,9 @@ class ElementInstance:
     mask: npt.NDArray[np.float32]
 
 SectionCallableInterface = Callable[[Union[int, str], PreprocessResponse], npt.NDArray[np.float32]]
-InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse], List[ElementInstance]]
+InstanceCallableInterface = Callable[[Union[int, str], PreprocessResponse, int], Optional[ElementInstance]]
+InstanceLengthCallableInterface = Callable[[Union[int, str], PreprocessResponse], int]
+
 
 MetadataSectionCallableInterface = Union[
     Callable[[Union[int, str], PreprocessResponse], int],
@@ -212,6 +212,11 @@ class MetadataHandler:
     metadata_type: Optional[Union[DatasetMetadataType, Dict[str, DatasetMetadataType]]] = None
 
 
+@dataclass
+class CustomLatentSpaceHandler:
+    function: SectionCallableInterface
+    name: str = 'custom_latent_space'
+
 @dataclass
 class PredictionTypeHandler:
     name: str
@@ -241,6 +246,7 @@ class DatasetIntegrationSetup:
     custom_loss_handlers: List[CustomLossHandler] = field(default_factory=list)
     metrics: List[MetricHandler] = field(default_factory=list)
     custom_layers: Dict[str, CustomLayerHandler] = field(default_factory=dict)
+    custom_latent_space: Optional[CustomLatentSpaceHandler] = None
 
 
 @dataclass
@@ -251,5 +257,6 @@ class DatasetSample:
     metadata_is_none: Dict[str, bool]
     index: Union[int, str]
     state: DataStateEnum
-    instance_masks: Optional[Dict[str, List[ElementInstance]]] = None
+    custom_latent_space: Optional[npt.NDArray[np.float32]] = None
+    instance_masks: Optional[Dict[str, ElementInstance]] = None
 

code_loader/inner_leap_binder/leapbinder.py

@@ -11,7 +11,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface
+    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
@@ -421,6 +421,19 @@ class LeapBinder:
         """
         self.setup_container.metadata.append(MetadataHandler(name, function, metadata_type))
 
+    def set_custom_latent_space(self, function: SectionCallableInterface) -> None:
+        """
+        Set a custom latent space function.
+
+        Args:
+        function (SectionCallableInterface): The metadata handler function.
+            This function receives:
+                subset (PreprocessResponse): The subset of the data.
+                index (int): The index of the sample within the subset.
+            This function should numpy float32 array contains the latent space vec of the sample.
+        """
+        self.setup_container.custom_latent_space = CustomLatentSpaceHandler(function)
+
     def set_custom_layer(self, custom_layer: Type[Any], name: str, inspect_layer: bool = False,
                          kernel_index: Optional[int] = None, use_custom_latent_space: bool = False) -> None:
         """

code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -8,7 +8,8 @@ import numpy.typing as npt
 from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs, \
     CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, CustomCallableInterface, \
     VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
-    ConfusionMatrixElement, SamplePreprocessResponse, PredictionTypeHandler, InstanceCallableInterface, ElementInstance
+    ConfusionMatrixElement, SamplePreprocessResponse, PredictionTypeHandler, InstanceCallableInterface, ElementInstance, \
+    InstanceLengthCallableInterface
 from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType
 from code_loader import leap_binder
 from code_loader.contract.mapping import NodeMapping, NodeMappingType, NodeConnection
@@ -443,6 +444,55 @@ def tensorleap_metadata(
     return decorating_function
 
 
+
+def tensorleap_custom_latent_space():
+    def decorating_function(user_function: SectionCallableInterface):
+        def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
+            assert isinstance(sample_id, (int, str)), \
+                (f'tensorleap_custom_latent_space validation failed: '
+                 f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_custom_latent_space validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
+            assert type(sample_id) == preprocess_response.sample_id_type, \
+                (f'tensorleap_custom_latent_space validation failed: '
+                 f'Argument sample_id should be as the same type as defined in the preprocess response '
+                 f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
+
+        def _validate_result(result):
+            assert isinstance(result, np.ndarray), \
+                (f'tensorleap_custom_loss validation failed: '
+                 f'The return type should be a numpy array. Got {type(result)}.')
+
+        def inner_without_validate(sample_id, preprocess_response):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(sample_id, preprocess_response)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
+        leap_binder.set_custom_latent_space(inner_without_validate)
+
+        def inner(sample_id, preprocess_response):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return None
+
+            _validate_input_args(sample_id, preprocess_response)
+
+            result = inner_without_validate(sample_id, preprocess_response)
+
+            _validate_result(result)
+            return result
+
+        return inner
+
+    return decorating_function
+
+
 def tensorleap_preprocess():
     def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
         leap_binder.set_preprocess(user_function)
@@ -480,29 +530,24 @@ def tensorleap_preprocess():
 
 
 def tensorleap_element_instance_preprocess(
-        instance_mask_encoder: Callable[[str, PreprocessResponse], List[ElementInstance]]):
+        instance_length_encoder: InstanceLengthCallableInterface):
     def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
         def user_function_instance() -> List[PreprocessResponse]:
             result = user_function()
             for preprocess_response in result:
                 sample_ids_to_instance_mappings = {}
                 instance_to_sample_ids_mappings = {}
-                instance_ids_to_names = {}
                 all_sample_ids = preprocess_response.sample_ids.copy()
                 for sample_id in preprocess_response.sample_ids:
-                    instances_masks = instance_mask_encoder(sample_id, preprocess_response)
-                    instances_ids = [f'{sample_id}_{instance_id}' for instance_id in range(len(instances_masks))]
+                    instances_length = instance_length_encoder(sample_id, preprocess_response)
+                    instances_ids = [f'{sample_id}_{instance_id}' for instance_id in range(instances_length)]
                     sample_ids_to_instance_mappings[sample_id] = instances_ids
                     instance_to_sample_ids_mappings[sample_id] = sample_id
-                    instance_names = [instance.name for instance in instances_masks]
-                    instance_ids_to_names[sample_id] = 'none'
-                    for instance_id, instance_name in zip(instances_ids, instance_names):
+                    for instance_id in instances_ids:
                         instance_to_sample_ids_mappings[instance_id] = sample_id
-                        instance_ids_to_names[instance_id] = instance_name
                     all_sample_ids.extend(instances_ids)
                 preprocess_response.sample_ids_to_instance_mappings = sample_ids_to_instance_mappings
                 preprocess_response.instance_to_sample_ids_mappings = instance_to_sample_ids_mappings
-                preprocess_response.instance_ids_to_names = instance_ids_to_names
                 preprocess_response.sample_ids = all_sample_ids
             return result
 
@@ -571,7 +616,7 @@ def tensorleap_unlabeled_preprocess():
 
 def tensorleap_instances_masks_encoder(name: str):
     def decorating_function(user_function: InstanceCallableInterface):
-        def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):
+        def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse, instance_id: int):
             assert isinstance(sample_id, str), \
                 (f'tensorleap_instances_masks_encoder validation failed: '
                  f'Argument sample_id should be str. Got {type(sample_id)}.')
@@ -582,11 +627,61 @@ def tensorleap_instances_masks_encoder(name: str):
                 (f'tensorleap_instances_masks_encoder validation failed: '
                  f'Argument sample_id should be as the same type as defined in the preprocess response '
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
+            assert isinstance(instance_id, int), \
+                (f'tensorleap_instances_masks_encoder validation failed: '
+                 f'Argument instance_id should be int. Got {type(instance_id)}.')
 
         def _validate_result(result):
-            assert isinstance(result, list), \
+            assert isinstance(result, ElementInstance) or (result is None), \
                 (f'tensorleap_instances_masks_encoder validation failed: '
-                 f'Unsupported return type. Should be a numpy array. Got {type(result)}.')
+                 f'Unsupported return type. Should be a ElementInstance or None. Got {type(result)}.')
+
+        def inner_without_validate(sample_id, preprocess_response, instance_id):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(sample_id, preprocess_response, instance_id)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
+        leap_binder.set_instance_masks(inner_without_validate, name)
+
+        def inner(sample_id, preprocess_response, instance_id):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return None
+
+            _validate_input_args(sample_id, preprocess_response, instance_id)
+
+            result = inner_without_validate(sample_id, preprocess_response, instance_id)
+
+            _validate_result(result)
+            return result
+
+        return inner
+
+    return decorating_function
+
+def tensorleap_instances_length_encoder(name: str):
+    def decorating_function(user_function: InstanceLengthCallableInterface):
+        def _validate_input_args(sample_id: str, preprocess_response: PreprocessResponse):
+            assert isinstance(sample_id, str), \
+                (f'tensorleap_instances_length_encoder validation failed: '
+                 f'Argument sample_id should be str. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_instances_length_encoder validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
+            assert type(sample_id) == preprocess_response.sample_id_type, \
+                (f'tensorleap_instances_length_encoder validation failed: '
+                 f'Argument sample_id should be as the same type as defined in the preprocess response '
+                 f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
+
+        def _validate_result(result):
+            assert isinstance(result, int), \
+                (f'tensorleap_instances_length_encoder validation failed: '
+                 f'Unsupported return type. Should be a int. Got {type(result)}.')
 
         def inner_without_validate(sample_id, preprocess_response):
             global _called_from_inside_tl_decorator
@@ -599,7 +694,7 @@ def tensorleap_instances_masks_encoder(name: str):
 
             return result
 
-        leap_binder.set_instance_masks(inner_without_validate, name)
+        # leap_binder.set_instance_masks(inner_without_validate, name). # TODO: do i need this?
 
         def inner(sample_id, preprocess_response):
             if os.environ.get(mapping_runtime_mode_env_var_mame):
@@ -616,7 +711,6 @@ def tensorleap_instances_masks_encoder(name: str):
 
     return decorating_function
 
-
 def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
     def decorating_function(user_function: SectionCallableInterface):
         for input_handler in leap_binder.setup_container.inputs:
@@ -626,8 +720,6 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
         if channel_dim <= 0 and channel_dim != -1:
             raise Exception(f"Channel dim for input {name} is expected to be either -1 or positive")
 
-        leap_binder.set_input(user_function, name, channel_dim=channel_dim)
-
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
             assert isinstance(sample_id, (int, str)), \
                 (f'tensorleap_input_encoder validation failed: '
@@ -650,15 +742,35 @@ def tensorleap_input_encoder(name: str, channel_dim=-1, model_input_index=None):
             assert channel_dim - 1 <= len(result.shape), (f'tensorleap_input_encoder validation failed: '
                                                           f'The channel_dim ({channel_dim}) should be <= to the rank of the resulting input rank ({len(result.shape)}).')
 
+        def inner_without_validate(sample_id, preprocess_response):
+
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(sample_id, preprocess_response)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
+        leap_binder.set_input(inner_without_validate, name, channel_dim=channel_dim)
+
+
         def inner(sample_id, preprocess_response):
             _validate_input_args(sample_id, preprocess_response)
-            result = user_function(sample_id, preprocess_response)
+
+            result = inner_without_validate(sample_id, preprocess_response)
+
             _validate_result(result)
 
             if _called_from_inside_tl_decorator == 0:
                 result = np.expand_dims(result, axis=0)
+
             return result
 
+
+
         node_mapping_type = NodeMappingType.Input
         if model_input_index is not None:
             node_mapping_type = NodeMappingType(f'Input{str(model_input_index)}')
@@ -695,8 +807,6 @@ def tensorleap_gt_encoder(name: str):
                 raise Exception(f'GT with name {name} already exists. '
                                 f'Please choose another')
 
-        leap_binder.set_ground_truth(user_function, name)
-
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
             assert isinstance(sample_id, (int, str)), \
                 (f'tensorleap_gt_encoder validation failed: '
@@ -717,13 +827,30 @@ def tensorleap_gt_encoder(name: str):
                 (f'tensorleap_gt_encoder validation failed: '
                  f'The return type should be a numpy array of type float32. Got {result.dtype}.')
 
+        def inner_without_validate(sample_id, preprocess_response):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(sample_id, preprocess_response)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
+        leap_binder.set_ground_truth(inner_without_validate, name)
+
+
         def inner(sample_id, preprocess_response):
             _validate_input_args(sample_id, preprocess_response)
-            result = user_function(sample_id, preprocess_response)
+
+            result = inner_without_validate(sample_id, preprocess_response)
+
             _validate_result(result)
 
             if _called_from_inside_tl_decorator == 0:
                 result = np.expand_dims(result, axis=0)
+
             return result
 
         inner.node_mapping = NodeMapping(name, NodeMappingType.GroundTruth)
@@ -760,14 +887,8 @@ def tensorleap_custom_loss(name: str, connects_to=None):
                                 f'Please choose another')
 
         valid_types = (np.ndarray, SamplePreprocessResponse)
-        try:
-            import tensorflow as tf
-            valid_types = (np.ndarray, SamplePreprocessResponse, tf.Tensor)
-        except ImportError:
-            pass
 
         def _validate_input_args(*args, **kwargs):
-
             for i, arg in enumerate(args):
                 if isinstance(arg, list):
                     for y, elem in enumerate(arg):
@@ -786,10 +907,11 @@ def tensorleap_custom_loss(name: str, connects_to=None):
                                                           f'Argument #{_arg_name} should be a numpy array. Got {type(arg)}.')
 
         def _validate_result(result):
-            assert isinstance(result, valid_types), \
+            assert isinstance(result, np.ndarray), \
                 (f'tensorleap_custom_loss validation failed: '
                  f'The return type should be a numpy array. Got {type(result)}.')
 
+
         @functools.wraps(user_function)
         def inner_without_validate(*args, **kwargs):
             global _called_from_inside_tl_decorator

code_loader/leaploader.py

@@ -16,7 +16,7 @@ from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandle
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
     MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
-    ElementInstance
+    ElementInstance, custom_latent_space_attribute
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -34,7 +34,7 @@ class LeapLoader(LeapLoaderBase):
         super().__init__(code_path, code_entry_name)
 
         self._preprocess_result_cached = None
-        
+
         try:
             from code_loader.mixpanel_tracker import track_code_loader_loaded
             track_code_loader_loaded({
@@ -77,6 +77,8 @@ class LeapLoader(LeapLoaderBase):
         file_path = Path(self.code_path, self.code_entry_name)
         append_path_recursively(str(file_path))
 
+        importlib.invalidate_caches()
+
         spec = importlib.util.spec_from_file_location(self.code_path, file_path)
         if spec is None or spec.loader is None:
             raise DatasetScriptException(f'Something is went wrong with spec file from: {file_path}')
@@ -155,37 +157,32 @@ class LeapLoader(LeapLoaderBase):
             for prediction_type in setup.prediction_types
         }
 
-    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
+    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: int = None) -> DatasetSample:
         self.exec_script()
         preprocess_result = self._preprocess_result()
         if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
             self._preprocess_result(update_unlabeled_preprocess=True)
 
         metadata, metadata_is_none = self._get_metadata(state, sample_id)
-        sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
-                               gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
-                               metadata=metadata,
-                               metadata_is_none=metadata_is_none,
-                               index=sample_id,
-                               state=state)
-        return sample
 
-    def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
-        self.exec_script()
-        preprocess_result = self._preprocess_result()
-        if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
-            self._preprocess_result(update_unlabeled_preprocess=True)
+        custom_latent_space = None
+        if global_leap_binder.setup_container.custom_latent_space is not None:
+            custom_latent_space = global_leap_binder.setup_container.custom_latent_space.function(sample_id,
+                                                                                                  preprocess_result[
+                                                                                                      state])
 
-        metadata, metadata_is_none = self._get_metadata(state, sample_id)
+        instance_mask = self._get_instances_masks(state, sample_id, instance_id)
         sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
                                gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
                                metadata=metadata,
                                metadata_is_none=metadata_is_none,
                                index=sample_id,
                                state=state,
-                               instance_masks=self._get_instances_masks(state, sample_id))
+                               custom_latent_space=custom_latent_space,
+                               instance_masks = instance_mask)
         return sample
 
+
     def check_dataset(self) -> DatasetIntegParseResult:
         test_payloads: List[DatasetTestResultPayload] = []
         setup_response = None
@@ -210,8 +207,6 @@ class LeapLoader(LeapLoaderBase):
                 general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                 is_valid = False
 
-
-
         print_log = stdout_steam.getvalue()
         is_valid_for_model = bool(global_leap_binder.setup_container.custom_layers)
         model_setup = self.get_model_setup_response()
@@ -476,12 +471,14 @@ class LeapLoader(LeapLoaderBase):
     def _get_inputs(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, npt.NDArray[np.float32]]:
         return self._get_dataset_handlers(global_leap_binder.setup_container.inputs, state, sample_id)
 
-    def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[str, List[ElementInstance]]:
+    def _get_instances_masks(self, state: DataStateEnum, sample_id: Union[int, str], instance_id: int) -> Optional[Dict[str, ElementInstance]]:
+        if instance_id is None:
+            return None
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
         result_agg = {}
         for handler in global_leap_binder.setup_container.instance_masks:
-            handler_result = handler.function(sample_id, preprocess_state)
+            handler_result = handler.function(sample_id, preprocess_state, instance_id)
             handler_name = handler.name
             result_agg[handler_name] = handler_result
         return result_agg
@@ -526,7 +523,8 @@ class LeapLoader(LeapLoaderBase):
 
         return converted_value, is_none
 
-    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
+    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[
+        Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
         result_agg = {}
         is_none = {}
         preprocess_result = self._preprocess_result()
@@ -555,7 +553,12 @@ class LeapLoader(LeapLoaderBase):
 
         return id_type
 
-    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str], Dict[str, str]]:
+    @lru_cache()
+    def has_custom_latent_space_decorator(self) -> bool:
+        self.exec_script()
+        return global_leap_binder.setup_container.custom_latent_space is not None
+
+    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
         """
         This Method get the data state and returns two dictionaries that holds the mapping of the sample ids to their
          instances and the other way around and the sample ids array.
@@ -568,4 +571,4 @@ class LeapLoader(LeapLoaderBase):
         """
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
-        return preprocess_state.sample_ids_to_instance_mappings, preprocess_state.instance_to_sample_ids_mappings, preprocess_state.instance_ids_to_names
+        return preprocess_state.sample_ids_to_instance_mappings, preprocess_state.instance_to_sample_ids_mappings

code_loader/leaploaderbase.py

@@ -65,11 +65,7 @@ class LeapLoaderBase:
         pass
 
     @abstractmethod
-    def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
-        pass
-
-    @abstractmethod
-    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str], Dict[str, str]]:
+    def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
         pass
 
     @abstractmethod
@@ -114,6 +110,10 @@ class LeapLoaderBase:
     def get_sample_id_type(self) -> Type:
         pass
 
+    @abstractmethod
+    def has_custom_latent_space_decorator(self) -> bool:
+        pass
+
     @abstractmethod
     def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
         pass

code_loader/mixpanel_tracker.py

@@ -94,10 +94,6 @@ class MixpanelTracker:
         Args:
             event_properties: Optional additional properties to include in the event
         """
-
-        if os.environ.get('IS_TENSORLEAP_PLATFORM') == 'true':
-            return
-            
         try:
             distinct_id = self._get_distinct_id()
             
@@ -110,8 +106,6 @@ class MixpanelTracker:
                 '$device_id': device_id,  # Always use device_id for $device_id
                 'python_version': f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
                 'platform': os.name,
-                'code_loader_version': '1.0',
-                'is_tensorleap_platform': os.environ.get('IS_TENSORLEAP_PLATFORM'),
             }
             
             if tensorleap_user_id:

code_loader/utils.py

@@ -18,12 +18,12 @@ def to_numpy_return_wrapper(encoder_function: SectionCallableInterface) -> Secti
 
     return numpy_encoder_function
 
-def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -> Callable[
-    [Union[int, str], PreprocessResponse], List[ElementInstance]]:
-    def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse) -> List[ElementInstance]:
-        result = encoder_function(idx, samples)
-        for res in result:
-            res.mask = np.array(res.mask)
+def to_numpy_return_masks_wrapper(encoder_function: InstanceCallableInterface) -> InstanceCallableInterface:
+    def numpy_encoder_function(idx: Union[int, str], samples: PreprocessResponse, element_idx: Union[int, str]) -> Union[ElementInstance, None]:
+        result = encoder_function(idx, samples, element_idx)
+        if result is None:
+            return None
+        result.mask = np.array(result.mask)
         return result
     return numpy_encoder_function
 

code_loader-1.0.121.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 TensorLeap
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{code_loader-1.0.119rc2.dist-info → code_loader-1.0.121.dist-info}/METADATA

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.119rc2
+Version: 1.0.121
 Summary: 
+Home-page: https://github.com/tensorleap/code-loader
 License: MIT
 Author: dorhar
 Author-email: doron.harnoy@tensorleap.ai
@@ -19,7 +20,6 @@ Requires-Dist: numpy (>=2.3.2,<3.0.0) ; python_version >= "3.11" and python_vers
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
-Project-URL: Homepage, https://github.com/tensorleap/code-loader
 Project-URL: Repository, https://github.com/tensorleap/code-loader
 Description-Content-Type: text/markdown
 

{code_loader-1.0.119rc2.dist-info → code_loader-1.0.121.dist-info}/RECORD

@@ -1,6 +1,7 @@
+LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=outxRQ0M-zMfV0QGVJmAed5qWfRmyD0TV6-goEGAzBw,406
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=gJsXu4zVAaiBlq6GJwPxfTD2e0gICTtI_6Ir61MRL48,8838
+code_loader/contract/datasetclasses.py,sha256=u0gfDDy02skhFG3ejJOxqxCnykhAcBPGJfv8Bi4s9eQ,8966
 code_loader/contract/enums.py,sha256=GEFkvUMXnCNt-GOoz7NJ9ecQZ2PPDettJNOsxsiM0wk,1622
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/mapping.py,sha256=e11h_sprwOyE32PcqgRq9JvyahQrPzwqgkhmbQLKLQY,1165
@@ -19,18 +20,18 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=0iHVHxC2NjfH7F0vQFVGy1e0llgKEyUHUHh3DdtqL70,32602
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=wjtk3TflrjJ8Y-OeuedVBD-09ZuOjIKGUjL7sMBU0fQ,41017
-code_loader/leaploader.py,sha256=rQRK1lyUPSpZiRs6lKUUxVJBCe3grEY_UxEMxSIpuxI,29709
-code_loader/leaploaderbase.py,sha256=lKdw2pd6H9hFsxVmc7jJMoZd_vlG5He1ooqT-cR_yq8,4496
-code_loader/mixpanel_tracker.py,sha256=b4yUE-F_lVeq5JlGjcbTCpdDVL34qYCC3WCElmeisEU,4865
+code_loader/inner_leap_binder/leapbinder.py,sha256=Q3D9yVM-GNEJfYRFvMV__BoZbcWOgnWKhrZXAv6Tu7o,33232
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=h_BTxp7kW71Wa60OfXBSTGaXvqZyZECtfCdC3j95_zw,45820
+code_loader/leaploader.py,sha256=e7oaxXuqNUWvon5h32pzzv7iHxpUbsVnQRWu9VgdTUc,29751
+code_loader/leaploaderbase.py,sha256=fFq9q8KM1wR9TsIfkdAkGKdz_Ss1JZmLIloMLaBAinQ,4433
+code_loader/mixpanel_tracker.py,sha256=U20vQXH8G7XIVXxcpQcVEZSuIwwGnyH5RMHXWZZG8HI,4639
 code_loader/plot_functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/plot_functions/plot_functions.py,sha256=xg6Gi4myTN9crq6JtyrhYI38HLXjPVJcbnI7CIy8f7w,14625
 code_loader/plot_functions/visualize.py,sha256=gsBAYYkwMh7jIpJeDMPS8G4CW-pxwx6LznoQIvi4vpo,657
-code_loader/utils.py,sha256=_j8b60pimoNAvWMRj7hEkkT6C76qES6cZoBFHpXHMxA,2698
+code_loader/utils.py,sha256=i1KOchLkieHfaVz6YskSIfKA45HcqAmGAK1F2Kcg38c,2724
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=onRnLE_TXfgLN4o52hQIOOhUcFexGlqJ3xSpQDVLuZM,2604
-code_loader-1.0.119rc2.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.119rc2.dist-info/METADATA,sha256=HFzUxae_sRJJHFJrzIVEKtpSCai_c8H4kbpDcxnrLOs,1105
-code_loader-1.0.119rc2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-code_loader-1.0.119rc2.dist-info/RECORD,,
+code_loader-1.0.121.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.121.dist-info/METADATA,sha256=mo_wEQJ-RufJvuLWjZtzhwFEu_JzP6jM1eRTHfkYMu0,1090
+code_loader-1.0.121.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.121.dist-info/RECORD,,

{code_loader-1.0.119rc2.dist-info → code_loader-1.0.121.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any