code-loader 1.0.117rc0__tar.gz → 1.0.118.dev1__tar.gz

{code_loader-1.0.117rc0 → code_loader-1.0.118.dev1}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.117rc0
+Version: 1.0.118.dev1
 Summary: 
+Home-page: https://github.com/tensorleap/code-loader
 License: MIT
 Author: dorhar
 Author-email: doron.harnoy@tensorleap.ai
@@ -19,7 +20,6 @@ Requires-Dist: numpy (>=2.3.2,<3.0.0) ; python_version >= "3.11" and python_vers
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
-Project-URL: Homepage, https://github.com/tensorleap/code-loader
 Project-URL: Repository, https://github.com/tensorleap/code-loader
 Description-Content-Type: text/markdown
 

{code_loader-1.0.117rc0 → code_loader-1.0.118.dev1}/code_loader/contract/datasetclasses.py

@@ -212,6 +212,11 @@ class MetadataHandler:
     metadata_type: Optional[Union[DatasetMetadataType, Dict[str, DatasetMetadataType]]] = None
 
 
+@dataclass
+class CustomLatentSpaceHandler:
+    function: SectionCallableInterface
+    name: str = 'custom_latent_space'
+
 @dataclass
 class PredictionTypeHandler:
     name: str
@@ -241,6 +246,7 @@ class DatasetIntegrationSetup:
     custom_loss_handlers: List[CustomLossHandler] = field(default_factory=list)
     metrics: List[MetricHandler] = field(default_factory=list)
     custom_layers: Dict[str, CustomLayerHandler] = field(default_factory=dict)
+    custom_latent_space: Optional[CustomLatentSpaceHandler] = None
 
 
 @dataclass
@@ -252,4 +258,5 @@ class DatasetSample:
     index: Union[int, str]
     state: DataStateEnum
     instance_masks: Optional[Dict[str, List[ElementInstance]]] = None
+    custom_latent_space: Optional[npt.NDArray[np.float32]] = None
 

{code_loader-1.0.117rc0 → code_loader-1.0.118.dev1}/code_loader/inner_leap_binder/leapbinder.py

@@ -11,7 +11,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse, \
-    ElementInstanceMasksHandler, InstanceCallableInterface
+    ElementInstanceMasksHandler, InstanceCallableInterface, CustomLatentSpaceHandler
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.mapping import NodeConnection, NodeMapping, NodeMappingType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
@@ -421,6 +421,19 @@ class LeapBinder:
         """
         self.setup_container.metadata.append(MetadataHandler(name, function, metadata_type))
 
+    def set_custom_latent_space(self, function: SectionCallableInterface) -> None:
+        """
+        Set a custom latent space function.
+
+        Args:
+        function (SectionCallableInterface): The metadata handler function.
+            This function receives:
+                subset (PreprocessResponse): The subset of the data.
+                index (int): The index of the sample within the subset.
+            This function should numpy float32 array contains the latent space vec of the sample.
+        """
+        self.setup_container.custom_latent_space = CustomLatentSpaceHandler(function)
+
     def set_custom_layer(self, custom_layer: Type[Any], name: str, inspect_layer: bool = False,
                          kernel_index: Optional[int] = None, use_custom_latent_space: bool = False) -> None:
         """

{code_loader-1.0.117rc0 → code_loader-1.0.118.dev1}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -443,6 +443,55 @@ def tensorleap_metadata(
     return decorating_function
 
 
+
+def tensorleap_custom_latent_space():
+    def decorating_function(user_function: SectionCallableInterface):
+        def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
+            assert isinstance(sample_id, (int, str)), \
+                (f'tensorleap_custom_latent_space validation failed: '
+                 f'Argument sample_id should be either int or str. Got {type(sample_id)}.')
+            assert isinstance(preprocess_response, PreprocessResponse), \
+                (f'tensorleap_custom_latent_space validation failed: '
+                 f'Argument preprocess_response should be a PreprocessResponse. Got {type(preprocess_response)}.')
+            assert type(sample_id) == preprocess_response.sample_id_type, \
+                (f'tensorleap_custom_latent_space validation failed: '
+                 f'Argument sample_id should be as the same type as defined in the preprocess response '
+                 f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
+
+        def _validate_result(result):
+            assert isinstance(result, np.ndarray), \
+                (f'tensorleap_custom_loss validation failed: '
+                 f'The return type should be a numpy array. Got {type(result)}.')
+
+        def inner_without_validate(sample_id, preprocess_response):
+            global _called_from_inside_tl_decorator
+            _called_from_inside_tl_decorator += 1
+
+            try:
+                result = user_function(sample_id, preprocess_response)
+            finally:
+                _called_from_inside_tl_decorator -= 1
+
+            return result
+
+        leap_binder.set_custom_latent_space(inner_without_validate)
+
+        def inner(sample_id, preprocess_response):
+            if os.environ.get(mapping_runtime_mode_env_var_mame):
+                return None
+
+            _validate_input_args(sample_id, preprocess_response)
+
+            result = inner_without_validate(sample_id, preprocess_response)
+
+            _validate_result(result)
+            return result
+
+        return inner
+
+    return decorating_function
+
+
 def tensorleap_preprocess():
     def decorating_function(user_function: Callable[[], List[PreprocessResponse]]):
         leap_binder.set_preprocess(user_function)
@@ -760,14 +809,8 @@ def tensorleap_custom_loss(name: str, connects_to=None):
                                 f'Please choose another')
 
         valid_types = (np.ndarray, SamplePreprocessResponse)
-        try:
-            import tensorflow as tf
-            valid_types = (np.ndarray, SamplePreprocessResponse, tf.Tensor)
-        except ImportError:
-            pass
 
         def _validate_input_args(*args, **kwargs):
-
             for i, arg in enumerate(args):
                 if isinstance(arg, list):
                     for y, elem in enumerate(arg):
@@ -786,10 +829,11 @@ def tensorleap_custom_loss(name: str, connects_to=None):
                                                           f'Argument #{_arg_name} should be a numpy array. Got {type(arg)}.')
 
         def _validate_result(result):
-            assert isinstance(result, valid_types), \
+            assert isinstance(result, np.ndarray), \
                 (f'tensorleap_custom_loss validation failed: '
                  f'The return type should be a numpy array. Got {type(result)}.')
 
+
         @functools.wraps(user_function)
         def inner_without_validate(*args, **kwargs):
             global _called_from_inside_tl_decorator

{code_loader-1.0.117rc0 → code_loader-1.0.118.dev1}/code_loader/leaploader.py

@@ -16,7 +16,7 @@ from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandle
     PreprocessResponse, VisualizerHandler, LeapData, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
     MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap, SamplePreprocessResponse, \
-    ElementInstance
+    ElementInstance, custom_latent_space_attribute
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
@@ -162,12 +162,18 @@ class LeapLoader(LeapLoaderBase):
             self._preprocess_result(update_unlabeled_preprocess=True)
 
         metadata, metadata_is_none = self._get_metadata(state, sample_id)
+
+        custom_latent_space = None
+        if global_leap_binder.setup_container.custom_latent_space is not None:
+            custom_latent_space = global_leap_binder.setup_container.custom_latent_space(sample_id, preprocess_result[state])
+
         sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
                                gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
                                metadata=metadata,
                                metadata_is_none=metadata_is_none,
                                index=sample_id,
-                               state=state)
+                               state=state,
+                               custom_latent_space=custom_latent_space)
         return sample
 
     def get_sample_with_masks(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:

{code_loader-1.0.117rc0 → code_loader-1.0.118.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.117rc0"
+version = "1.0.118.dev1"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"