code-loader 1.0.77.1__py3-none-any.whl → 1.0.78__py3-none-any.whl

code_loader/contract/datasetclasses.py

@@ -4,7 +4,8 @@ import re
 import numpy as np
 import numpy.typing as npt
 
-from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataType, ConfusionMatrixValue, MetricDirection
+from code_loader.contract.enums import DataStateType, DataStateEnum, LeapDataType, ConfusionMatrixValue, \
+    MetricDirection, DatasetMetadataType
 from code_loader.contract.visualizer_classes import LeapImage, LeapText, LeapGraph, LeapHorizontalBar, \
     LeapTextMask, LeapImageMask, LeapImageWithBBox, LeapImageWithHeatmap
 
@@ -196,6 +197,7 @@ class GroundTruthHandler(DatasetBaseHandler):
 class MetadataHandler:
     name: str
     function: MetadataSectionCallableInterface
+    metadata_type: Optional[Union[DatasetMetadataType, Dict[str, DatasetMetadataType]]] = None
 
 
 @dataclass
@@ -232,6 +234,7 @@ class DatasetIntegrationSetup:
 class DatasetSample:
     inputs: Dict[str, npt.NDArray[np.float32]]
     gt: Optional[Dict[str, npt.NDArray[np.float32]]]
-    metadata: Dict[str, Union[str, int, bool, float]]
+    metadata: Dict[str, Union[Optional[str], int, bool, Optional[float]]]
+    metadata_is_none: Dict[str, bool]
     index: Union[int, str]
     state: DataStateEnum

code_loader/inner_leap_binder/leapbinder.py

@@ -11,7 +11,7 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
     CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
     RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData, SamplePreprocessResponse
-from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
+from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection, DatasetMetadataType
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
 from code_loader.default_losses import loss_name_to_function
@@ -333,7 +333,8 @@ class LeapBinder:
 
         self._encoder_names.append(name)
 
-    def set_metadata(self, function: MetadataSectionCallableInterface, name: str) -> None:
+    def set_metadata(self, function: MetadataSectionCallableInterface, name: str,
+                     metadata_type: Optional[Union[DatasetMetadataType, Dict[str, DatasetMetadataType]]] = None) -> None:
         """
         Set the metadata handler function. This function is used for measuring and analyzing external variable values per sample, which is recommended for analysis within the Tensorleap platform.
 
@@ -368,7 +369,7 @@ class LeapBinder:
             leap_binder.set_metadata(metadata_handler_index, name='metadata_index')
             leap_binder.set_metadata(metadata_handler_image_mean, name='metadata_image_mean')
         """
-        self.setup_container.metadata.append(MetadataHandler(name, function))
+        self.setup_container.metadata.append(MetadataHandler(name, function, metadata_type))
 
     def set_custom_layer(self, custom_layer: Type[Any], name: str, inspect_layer: bool = False,
                          kernel_index: Optional[int] = None, use_custom_latent_space: bool = False) -> None:
@@ -465,23 +466,55 @@ class LeapBinder:
     @staticmethod
     def check_handler(
             preprocess_response: PreprocessResponse, test_result: List[DatasetTestResultPayload],
-            dataset_base_handler: Union[DatasetBaseHandler, MetadataHandler]) -> List[DatasetTestResultPayload]:
+            dataset_base_handler: Union[DatasetBaseHandler, MetadataHandler], state: DataStateEnum) -> List[DatasetTestResultPayload]:
         assert preprocess_response.sample_ids is not None
         raw_result = dataset_base_handler.function(preprocess_response.sample_ids[0], preprocess_response)
         handler_type = 'metadata' if isinstance(dataset_base_handler, MetadataHandler) else None
-        if isinstance(dataset_base_handler, MetadataHandler) and isinstance(raw_result, dict):
-            metadata_test_result_payloads = [
-                DatasetTestResultPayload(f'{dataset_base_handler.name}_{single_metadata_name}')
-                for single_metadata_name, single_metadata_result in raw_result.items()
-            ]
-            for i, (single_metadata_name, single_metadata_result) in enumerate(raw_result.items()):
-                metadata_test_result = metadata_test_result_payloads[i]
-
-                result_shape = get_shape(single_metadata_result)
-                metadata_test_result.shape = result_shape
-                metadata_test_result.raw_result = single_metadata_result
-                metadata_test_result.handler_type = handler_type
-            test_result = metadata_test_result_payloads
+        if isinstance(dataset_base_handler, MetadataHandler):
+            if isinstance(raw_result, dict):
+                metadata_test_result_payloads = [
+                    DatasetTestResultPayload(f'{dataset_base_handler.name}_{single_metadata_name}')
+                    for single_metadata_name, single_metadata_result in raw_result.items()
+                ]
+                for i, (single_metadata_name, single_metadata_result) in enumerate(raw_result.items()):
+                    metadata_test_result = metadata_test_result_payloads[i]
+
+                    metadata_type = None
+                    if single_metadata_result is None:
+                        if state != DataStateEnum.training and test_result[i].name == f'{dataset_base_handler.name}_{single_metadata_name}':
+                            metadata_test_result_payloads[i] = test_result[i]
+                            continue
+
+                        if dataset_base_handler.metadata_type is None:
+                            raise Exception(f"Metadata {single_metadata_name} is None and no metadata type is provided")
+                        elif isinstance(dataset_base_handler.metadata_type, dict):
+                            if single_metadata_name not in dataset_base_handler.metadata_type:
+                                raise Exception(f"Metadata {single_metadata_name} is None and no metadata type is provided")
+                            metadata_type = dataset_base_handler.metadata_type[single_metadata_name]
+                        else:
+                            raise Exception(f"Metadata {single_metadata_name} is None and no metadata type is provided")
+
+                    result_shape = get_shape(single_metadata_result)
+                    metadata_test_result.shape = result_shape
+                    metadata_test_result.raw_result = (
+                        single_metadata_result) if single_metadata_result is not None else metadata_type
+                    metadata_test_result.handler_type = handler_type
+                test_result = metadata_test_result_payloads
+            else:
+                if raw_result is None:
+                    if state != DataStateEnum.training:
+                        return test_result
+
+                    if dataset_base_handler.metadata_type is None:
+                        raise Exception(f"Metadata {dataset_base_handler.name} is None and no metadata type is provided")
+                    elif isinstance(dataset_base_handler.metadata_type, dict):
+                        raise Exception(f"Metadata {dataset_base_handler.name} is None and no metadata type is provided")
+                    metadata_type = dataset_base_handler.metadata_type
+
+                result_shape = get_shape(raw_result)
+                test_result[0].shape = result_shape
+                test_result[0].raw_result = raw_result if raw_result is not None else metadata_type
+                test_result[0].handler_type = handler_type
         else:
             result_shape = get_shape(raw_result)
             test_result[0].shape = result_shape
@@ -500,7 +533,7 @@ class LeapBinder:
             for state, preprocess_response in preprocess_result.items():
                 if state == DataStateEnum.unlabeled and isinstance(dataset_base_handler, GroundTruthHandler):
                     continue
-                self.check_handler(preprocess_response, test_result, dataset_base_handler)
+                self.check_handler(preprocess_response, test_result, dataset_base_handler, state)
 
     def check(self) -> None:
         preprocess_result = self.get_preprocess_result()

code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -9,7 +9,7 @@ from code_loader.contract.datasetclasses import CustomCallableInterfaceMultiArgs
     CustomMultipleReturnCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, CustomCallableInterface, \
     VisualizerCallableInterface, MetadataSectionCallableInterface, PreprocessResponse, SectionCallableInterface, \
     ConfusionMatrixElement, SamplePreprocessResponse
-from code_loader.contract.enums import MetricDirection, LeapDataType
+from code_loader.contract.enums import MetricDirection, LeapDataType, DatasetMetadataType
 from code_loader import leap_binder
 from code_loader.contract.visualizer_classes import LeapImage, LeapImageMask, LeapTextMask, LeapText, LeapGraph, \
     LeapHorizontalBar, LeapImageWithBBox, LeapImageWithHeatmap
@@ -160,14 +160,15 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
     return decorating_function
 
 
-def tensorleap_metadata(name: str):
+def tensorleap_metadata(
+        name: str, metadata_type: Optional[Union[DatasetMetadataType, Dict[str, DatasetMetadataType]]] = None):
     def decorating_function(user_function: MetadataSectionCallableInterface):
         for metadata_handler in leap_binder.setup_container.metadata:
             if metadata_handler.name == name:
                 raise Exception(f'Metadata with name {name} already exists. '
                                 f'Please choose another')
 
-        leap_binder.set_metadata(user_function, name)
+        leap_binder.set_metadata(user_function, name, metadata_type)
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
             assert isinstance(sample_id, (int, str)), \
@@ -182,7 +183,7 @@ def tensorleap_metadata(name: str):
                  f'{preprocess_response.sample_id_type}. Got {type(sample_id)}.')
 
         def _validate_result(result):
-            supported_result_types = (int, str, bool, float, dict, np.floating,
+            supported_result_types = (type(None), int, str, bool, float, dict, np.floating,
                                       np.bool_, np.unsignedinteger, np.signedinteger, np.integer)
             assert isinstance(result, supported_result_types), \
                 (f'tensorleap_metadata validation failed: '

code_loader/leaploader.py

@@ -6,7 +6,7 @@ import sys
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union, Any, Type, Optional, Callable
+from typing import Dict, List, Iterable, Union, Any, Type, Optional, Callable, Tuple
 
 import numpy as np
 import numpy.typing as npt
@@ -140,9 +140,11 @@ class LeapLoader(LeapLoaderBase):
         if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
             self._preprocess_result(update_unlabeled_preprocess=True)
 
+        metadata, metadata_is_none = self._get_metadata(state, sample_id)
         sample = DatasetSample(inputs=self._get_inputs(state, sample_id),
                                gt=None if state == DataStateEnum.unlabeled else self._get_gt(state, sample_id),
-                               metadata=self._get_metadata(state, sample_id),
+                               metadata=metadata,
+                               metadata_is_none=metadata_is_none,
                                index=sample_id,
                                state=state)
         return sample
@@ -210,7 +212,7 @@ class LeapLoader(LeapLoaderBase):
                 state_name = state.name
                 try:
                     test_result = global_leap_binder.check_handler(
-                        preprocess_response, test_result, dataset_base_handler)
+                        preprocess_response, test_result, dataset_base_handler, state)
                 except Exception as e:
                     line_number, file_name, stacktrace = get_root_exception_file_and_line_number()
                     test_result[0].display[
@@ -339,6 +341,11 @@ class LeapLoader(LeapLoaderBase):
                 continue
             if hasattr(handler_test_payload.raw_result, 'tolist'):
                 handler_test_payload.raw_result = handler_test_payload.raw_result.tolist()
+            if isinstance(handler_test_payload.raw_result, DatasetMetadataType):
+                dataset_metadata_type = handler_test_payload.raw_result
+                metadata_instances.append(DatasetMetadataInstance(name=handler_test_payload.name,
+                                                                  type=dataset_metadata_type))
+                continue
             metadata_type = type(handler_test_payload.raw_result)
             if metadata_type == int or isinstance(handler_test_payload.raw_result,
                                                   (np.unsignedinteger, np.signedinteger)):
@@ -438,7 +445,7 @@ class LeapLoader(LeapLoaderBase):
         }
         return metadata_name_to_type
 
-    def _convert_metadata_to_correct_type(self, metadata_name: str, value: Any) -> Any:
+    def _convert_metadata_to_correct_type(self, metadata_name: str, value: Any) -> Tuple[Any, bool]:
         metadata_name_to_type = self._metadata_name_to_type()
         metadata_type_to_python_type = {
             DatasetMetadataType.float: float,
@@ -447,22 +454,26 @@ class LeapLoader(LeapLoaderBase):
             DatasetMetadataType.int: int
         }
         metadata_type_to_default_value = {
-            DatasetMetadataType.float: -1,
-            DatasetMetadataType.string: "",
+            DatasetMetadataType.float: -1.0,
+            DatasetMetadataType.string: 'None',
             DatasetMetadataType.boolean: False,
             DatasetMetadataType.int: -1
         }
 
         try:
+            is_none = False
+            if value is None:
+                raise ValueError()
             converted_value = metadata_type_to_python_type[metadata_name_to_type[metadata_name]](value)
         except ValueError:
+            is_none = True
             converted_value = metadata_type_to_default_value[metadata_name_to_type[metadata_name]]
 
-        return converted_value
+        return converted_value, is_none
 
-    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Dict[
-        str, Union[str, int, bool, float]]:
+    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
         result_agg = {}
+        is_none = {}
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
         for handler in global_leap_binder.setup_container.metadata:
@@ -470,13 +481,14 @@ class LeapLoader(LeapLoaderBase):
             if isinstance(handler_result, dict):
                 for single_metadata_name, single_metadata_result in handler_result.items():
                     handler_name = f'{handler.name}_{single_metadata_name}'
-                    result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name,
-                                                                                      single_metadata_result)
+                    result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
+                        handler_name, single_metadata_result)
             else:
                 handler_name = handler.name
-                result_agg[handler_name] = self._convert_metadata_to_correct_type(handler_name, handler_result)
+                result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
+                    handler_name, handler_result)
 
-        return result_agg
+        return result_agg, is_none
 
     @lru_cache()
     def get_sample_id_type(self) -> Type:

{code_loader-1.0.77.1.dist-info → code_loader-1.0.78.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.77.1
+Version: 1.0.78
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.77.1.dist-info → code_loader-1.0.78.dist-info}/RECORD

@@ -1,7 +1,7 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=t1pvW68dVIbYB0bllqxHXO6jOgHCJI5NGn9rSrYC6Vw,7722
+code_loader/contract/datasetclasses.py,sha256=BjT7NoOBgWtBXtMRedKwwnC-vbHg-KHKqFL11GdgUx8,7902
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=RSx9m_R3LawhK5o1nAcO3hfp2F9oJYtxZr_bpP3bTmw,4005
@@ -19,14 +19,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=0l9zjlF27tZwg6SnpyqVoAAgf9QHQcKpR9lg7vho2Xw,27065
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=mkgQF5YBMlw5_1JXj7LnhDQpRqlq-8kvP7uFvB3DTRo,21904
-code_loader/leaploader.py,sha256=KkAPKyzZxZYI135_67q63ICVVERu7WH60liB0pboA-c,24983
+code_loader/inner_leap_binder/leapbinder.py,sha256=3C8Vzk-O2opVQ5FU16uX7GM57RX64mziMiyW6h3fRgo,29367
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=HUIVrI-I0aB-nQl7lZErE5vuhml7A8MMGaw_R4U4QCk,22053
+code_loader/leaploader.py,sha256=AUx6iqJK9DzIrdDGvfM3cWNejoK7aDTsU_ShhphVrfM,25679
 code_loader/leaploaderbase.py,sha256=VH0vddRmkqLtcDlYPCO7hfz1_VbKo43lUdHDAbd4iJc,4198
 code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=Ffx5VHVOe5ujBOsjBSxN_aIEVwFSQ6gbhTMG5aUS-po,2305
-code_loader-1.0.77.1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.77.1.dist-info/METADATA,sha256=ko-pD3mbQoKtxXhbVRaxEsrAOWpqYacD-791pEGkabU,851
-code_loader-1.0.77.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-code_loader-1.0.77.1.dist-info/RECORD,,
+code_loader-1.0.78.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.78.dist-info/METADATA,sha256=EfiPvcKzkE4v7DblDFb6-kpVDBXz_pquAep0j7FTAQ4,849
+code_loader-1.0.78.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.78.dist-info/RECORD,,

{code_loader-1.0.77.1.dist-info → code_loader-1.0.78.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any