code-loader 1.0.150.dev1__tar.gz → 1.0.151__tar.gz

{code_loader-1.0.150.dev1 → code_loader-1.0.151}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.150.dev1
+Version: 1.0.151
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.150.dev1 → code_loader-1.0.151}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -132,8 +132,7 @@ def validate_output_structure(result, func_name: str, expected_type_name="np.nda
 
 
 def batch_warning(result, func_name):
-    return
-    if result.shape[0] == 1:
+    if len(result.shape) > 0 and result.shape[0] == 1:
         warnings.warn(
             f"{func_name} warning: Tensorleap will add a batch dimension at axis 0 to the output of {func_name}, "
             f"although the detected size of axis 0 is already 1. "

{code_loader-1.0.150.dev1 → code_loader-1.0.151}/code_loader/leaploader.py

@@ -163,7 +163,7 @@ class LeapLoader(LeapLoaderBase):
         if state == DataStateEnum.unlabeled and sample_id not in preprocess_result[state].sample_ids:
             self._preprocess_result(update_unlabeled_preprocess=True)
 
-        metadata, metadata_is_none = self._get_metadata(state, sample_id)
+        metadata, metadata_is_none = self.get_metadata(state, sample_id)
 
         custom_latent_space = None
         if global_leap_binder.setup_container.custom_latent_space is not None:
@@ -521,21 +521,38 @@ class LeapLoader(LeapLoaderBase):
 
         return converted_value, is_none
 
-    def _get_metadata(self, state: DataStateEnum, sample_id: Union[int, str]) -> Tuple[
+    def get_metadata(self, state: DataStateEnum, sample_id: Union[int, str], requested_metadata_names: Optional[List[str]] = None) -> Tuple[
         Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
+
+        def is_metadata_name_starts_with_handler_name(_handler):
+            for metadata_name in requested_metadata_names:
+                if metadata_name.startswith(_handler.name + '_') or metadata_name == _handler.name:
+                    return True
+            return False
+
         result_agg = {}
         is_none = {}
         preprocess_result = self._preprocess_result()
         preprocess_state = preprocess_result[state]
         for handler in global_leap_binder.setup_container.metadata:
+            if requested_metadata_names:
+                if not is_metadata_name_starts_with_handler_name(handler):
+                    continue
+
             handler_result = handler.function(sample_id, preprocess_state)
             if isinstance(handler_result, dict):
                 for single_metadata_name, single_metadata_result in handler_result.items():
                     handler_name = f'{handler.name}_{single_metadata_name}'
+                    if requested_metadata_names:
+                        if handler_name not in requested_metadata_names:
+                            continue
                     result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
                         handler_name, single_metadata_result)
             else:
                 handler_name = handler.name
+                if requested_metadata_names:
+                    if handler_name not in requested_metadata_names:
+                        continue
                 result_agg[handler_name], is_none[handler_name] = self._convert_metadata_to_correct_type(
                     handler_name, handler_result)
 

{code_loader-1.0.150.dev1 → code_loader-1.0.151}/code_loader/leaploaderbase.py

@@ -68,6 +68,24 @@ class LeapLoaderBase:
     def get_instances_data(self, state: DataStateEnum) -> Tuple[Dict[str, List[str]], Dict[str, str]]:
         pass
 
+    def get_metadata_multiple_samples(self, state: DataStateEnum, sample_ids: Union[List[int], List[str]],
+                                      requested_metadata_names: Optional[List[str]] = None
+                                      ) -> Tuple[Dict[str, Union[List[str], List[int], List[bool],
+    List[float]]], Dict[str, List[bool]]]:
+        aggregated_results: Dict[str, List[Union[str, int, bool, float]]] = {}
+        aggregated_is_none: Dict[str, List[bool]] = {}
+        sample_id_type = self.get_sample_id_type()
+        for sample_id in sample_ids:
+            sample_id = sample_id_type(sample_id)
+            metadata_result, is_none_result = self.get_metadata(state, sample_id, requested_metadata_names)
+            for metadata_name, metadata_value in metadata_result.items():
+                if metadata_name not in aggregated_results:
+                    aggregated_results[metadata_name] = []
+                    aggregated_is_none[metadata_name] = []
+                aggregated_results[metadata_name].append(metadata_value)
+                aggregated_is_none[metadata_name].append(is_none_result[metadata_name])
+        return aggregated_results, aggregated_is_none
+
     @abstractmethod
     def check_dataset(self) -> DatasetIntegParseResult:
         pass
@@ -87,6 +105,13 @@ class LeapLoaderBase:
                         input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
         pass
 
+    @abstractmethod
+    def get_metadata(
+            self, state: DataStateEnum, sample_id: Union[int, str],
+            requested_metadata_names: Optional[List[str]] = None
+    ) -> Tuple[Dict[str, Union[str, int, bool, float]], Dict[str, bool]]:
+        pass
+
     @abstractmethod
     def run_heatmap_visualizer(self, visualizer_name: str, sample_ids: np.array, state: DataStateEnum,
                                input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]

{code_loader-1.0.150.dev1 → code_loader-1.0.151}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.150.dev1"
+version = "1.0.151"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"