code-loader 0.2.87__tar.gz → 0.2.89__tar.gz

{code_loader-0.2.87 → code_loader-0.2.89}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 0.2.87
+Version: 0.2.89
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-0.2.87 → code_loader-0.2.89}/code_loader/contract/datasetclasses.py

@@ -21,9 +21,13 @@ SectionCallableInterface = Callable[[int, PreprocessResponse], npt.NDArray[np.fl
 
 MetadataSectionCallableInterface = Union[
     Callable[[int, PreprocessResponse], int],
+    Callable[[int, PreprocessResponse], Dict[str, int]],
     Callable[[int, PreprocessResponse], str],
+    Callable[[int, PreprocessResponse], Dict[str, str]],
     Callable[[int, PreprocessResponse], bool],
-    Callable[[int, PreprocessResponse], float]
+    Callable[[int, PreprocessResponse], Dict[str, bool]],
+    Callable[[int, PreprocessResponse], float],
+    Callable[[int, PreprocessResponse], Dict[str, float]]
 ]
 
 
@@ -66,6 +70,7 @@ class ConfusionMatrixElement:
 ConfusionMatrixCallableInterface = Callable[[tf.Tensor, tf.Tensor], List[List[ConfusionMatrixElement]]]
 
 CustomCallableInterfaceMultiArgs = Callable[..., tf.Tensor]
+CustomMultipleReturnCallableInterfaceMultiArgs = Callable[..., Dict[str, tf.Tensor]]
 ConfusionMatrixCallableInterfaceMultiArgs = Callable[..., List[List[ConfusionMatrixElement]]]
 MetricCallableReturnType = Union[tf.Tensor, List[List[ConfusionMatrixElement]]]
 
@@ -113,7 +118,6 @@ class GroundTruthHandler(DatasetBaseHandler):
 class MetadataHandler:
     name: str
     function: MetadataSectionCallableInterface
-    type: DatasetMetadataType
 
 
 @dataclass

{code_loader-0.2.87 → code_loader-0.2.89}/code_loader/contract/responsedataclasses.py

@@ -87,6 +87,8 @@ class DatasetTestResultPayload:
     display: Dict[str, str] = field(default_factory=dict)
     is_passed: bool = True
     shape: Optional[List[int]] = None
+    raw_result = None
+    handler_type = None
 
 
 @dataclass

{code_loader-0.2.87 → code_loader-0.2.89}/code_loader/leap_binder/leapbinder.py

@@ -10,8 +10,9 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     GroundTruthHandler, MetadataHandler, DatasetIntegrationSetup, VisualizerHandler, PreprocessResponse, \
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
-    ConfusionMatrixCallableInterface, CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs
-from code_loader.contract.enums import DatasetMetadataType, LeapDataType, MetricEnum
+    CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, \
+    CustomMultipleReturnCallableInterfaceMultiArgs
+from code_loader.contract.enums import LeapDataType
 from code_loader.metrics.default_metrics import metrics_names_to_functions
 from code_loader.utils import to_numpy_return_wrapper
 from code_loader.visualizers.default_visualizers import DefaultVisualizer, \
@@ -86,18 +87,15 @@ class LeapBinder:
 
     @typechecked
     def add_custom_metric(self,
-                          function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs],
+                          function: Union[CustomCallableInterfaceMultiArgs,
+                                          CustomMultipleReturnCallableInterfaceMultiArgs,
+                                          ConfusionMatrixCallableInterfaceMultiArgs],
                           name: str) -> None:
         arg_names = inspect.getfullargspec(function)[0]
         self.setup_container.metrics.append(MetricHandler(name, function, arg_names))
 
     @typechecked
-    def add_prediction(self, name: str, labels: List[str], metrics: Optional[List[MetricEnum]] = None,
-                       custom_metrics: Optional[
-                           List[Union[CustomCallableInterface, ConfusionMatrixCallableInterface]]] = None) -> None:
-        if metrics or custom_metrics:
-            raise DeprecationWarning("Adding metrics on 'leap_binder.add_prediction' method is deprecated."
-                                     "Please update the leap script and use metric block instead.")
+    def add_prediction(self, name: str, labels: List[str]) -> None:
         self.setup_container.prediction_types.append(PredictionTypeHandler(name, labels))
 
     @typechecked
@@ -108,9 +106,8 @@ class LeapBinder:
         self._encoder_names.append(name)
 
     @typechecked
-    def set_metadata(self, function: MetadataSectionCallableInterface, metadata_type: DatasetMetadataType,
-                     name: str) -> None:
-        self.setup_container.metadata.append(MetadataHandler(name, function, metadata_type))
+    def set_metadata(self, function: MetadataSectionCallableInterface, name: str) -> None:
+        self.setup_container.metadata.append(MetadataHandler(name, function))
 
     @typechecked
     def set_custom_layer(self, custom_layer: Type[tf.keras.layers.Layer], name: str) -> None:

{code_loader-0.2.87 → code_loader-0.2.89}/code_loader/leap_loader_parallelized_base.py

@@ -32,13 +32,13 @@ class LeapLoaderParallelizedBase(ABC):
         self._generate_inputs_thread: Optional[Thread] = None
         self._should_stop_thread = False
 
-    def _calculate_n_workers_by_hardware(self) -> int:
+    def _calculate_n_workers_bpyproject.tomly_hardware(self) -> int:
         p = psutil.Process(self.processes[0].pid)
         memory_usage_in_bytes = p.memory_info().rss
         total_memory_in_bytes = psutil.virtual_memory().total
 
         n_workers = min(int(multiprocessing.cpu_count()),
-                        int(total_memory_in_bytes * 0.7 / memory_usage_in_bytes))
+                        int(total_memory_in_bytes * 0.5 / memory_usage_in_bytes))
         n_workers = max(n_workers, 1)
         return n_workers
 

{code_loader-0.2.87 → code_loader-0.2.89}/code_loader/leaploader.py

@@ -22,7 +22,7 @@ except Exception as e:
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, InputHandler, \
     GroundTruthHandler, PreprocessResponse, VisualizerHandler, VisualizerCallableReturnType, CustomLossHandler, \
     PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler
-from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType
+from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
@@ -44,6 +44,12 @@ class LeapLoader:
                 torch.cuda.is_available = lambda: False
 
             self.evaluate_module()
+        except TypeError as e:
+            import traceback
+            if "leap_binder.set_metadata(" in traceback.format_exc(5):
+                raise DeprecationWarning(
+                    "Please remove the metadata_type on leap_binder.set_metadata in your dataset script")
+            raise DatasetScriptException(getattr(e, 'message', repr(e))) from e
         except Exception as e:
             raise DatasetScriptException(getattr(e, 'message', repr(e))) from e
 
@@ -133,7 +139,7 @@ class LeapLoader:
                 handlers_test_payloads = self._check_handlers()
                 test_payloads.extend(handlers_test_payloads)
                 is_valid = all([payload.is_passed for payload in test_payloads])
-                setup_response = self.get_dataset_setup_response()
+                setup_response = self.get_dataset_setup_response(handlers_test_payloads)
             except DatasetScriptException as e:
                 line_number = get_root_exception_line_number()
                 general_error = f"Something went wrong, {repr(e.__cause__)} line number: {line_number}"
@@ -180,26 +186,42 @@ class LeapLoader:
         idx = 0
         dataset_base_handlers: List[Union[DatasetBaseHandler, MetadataHandler]] = self._get_all_dataset_base_handlers()
         for dataset_base_handler in dataset_base_handlers:
-            test_result = DatasetTestResultPayload(dataset_base_handler.name)
+            test_result = [DatasetTestResultPayload(dataset_base_handler.name)]
             for state, preprocess_response in preprocess_result.items():
                 if state == DataStateEnum.unlabeled and isinstance(dataset_base_handler, GroundTruthHandler):
                     continue
                 state_name = state.name
                 try:
                     raw_result = dataset_base_handler.function(idx, preprocess_response)
-                    result_shape = get_shape(raw_result)
-                    test_result.shape = result_shape
-
-                    # setting shape in setup for all encoders
-                    if isinstance(dataset_base_handler, (InputHandler, GroundTruthHandler)):
-                        dataset_base_handler.shape = result_shape
+                    handler_type = 'metadata' if isinstance(dataset_base_handler, MetadataHandler) else None
+                    if isinstance(dataset_base_handler, MetadataHandler) and isinstance(raw_result, dict):
+                        metadata_test_result_payloads = [
+                            DatasetTestResultPayload(f'{dataset_base_handler.name}_{single_metadata_name}')
+                            for single_metadata_name, single_metadata_result in raw_result.items()
+                        ]
+                        for i, (single_metadata_name, single_metadata_result) in enumerate(raw_result.items()):
+                            metadata_test_result = metadata_test_result_payloads[i]
+                            result_shape = get_shape(single_metadata_result)
+                            metadata_test_result.shape = result_shape
+                            metadata_test_result.raw_result = single_metadata_result
+                            metadata_test_result.handler_type = handler_type
+                        test_result = metadata_test_result_payloads
+                    else:
+                        result_shape = get_shape(raw_result)
+                        test_result[0].shape = result_shape
+                        test_result[0].raw_result = raw_result
+                        test_result[0].handler_type = handler_type
+
+                        # setting shape in setup for all encoders
+                        if isinstance(dataset_base_handler, (InputHandler, GroundTruthHandler)):
+                            dataset_base_handler.shape = result_shape
 
                 except Exception as e:
                     line_number = get_root_exception_line_number()
-                    test_result.display[state_name] = f"{repr(e)} line number: {line_number}"
-                    test_result.is_passed = False
+                    test_result[0].display[state_name] = f"{repr(e)} line number: {line_number}"
+                    test_result[0].is_passed = False
 
-            result_payloads.append(test_result)
+            result_payloads.extend(test_result)
 
         return result_payloads
 
@@ -224,7 +246,7 @@ class LeapLoader:
         return heatmap_function(**input_tensors_by_arg_name)
 
     @staticmethod
-    def get_dataset_setup_response() -> DatasetSetup:
+    def get_dataset_setup_response(handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
         setup = global_leap_binder.setup_container
         assert setup.preprocess is not None
 
@@ -251,8 +273,26 @@ class LeapLoader:
             ground_truths.append(
                 DatasetOutputInstance(name=gt.name, shape=gt.shape))
 
-        metadata = [DatasetMetadataInstance(name=metadata.name, type=metadata.type)
-                    for metadata in setup.metadata]
+        metadata_instances = []
+        for handler_test_payload in handlers_test_payloads:
+            if handler_test_payload.handler_type != 'metadata':
+                continue
+            if hasattr(handler_test_payload.raw_result, 'tolist'):
+                handler_test_payload.raw_result = handler_test_payload.raw_result.tolist()
+            metadata_type = type(handler_test_payload.raw_result)
+            if metadata_type == int:
+                metadata_type = float
+            if metadata_type == str:
+                dataset_metadata_type = DatasetMetadataType.string
+            elif metadata_type == bool:
+                dataset_metadata_type = DatasetMetadataType.boolean
+            elif metadata_type == float:
+                dataset_metadata_type = DatasetMetadataType.float
+            else:
+                raise Exception(f"Unsupported return type of metadata {handler_test_payload.name}."
+                                f"The return type should be one of [int, float, str, bool]. Got {metadata_type}")
+            metadata_instances.append(DatasetMetadataInstance(name=handler_test_payload.name,
+                                                              type=dataset_metadata_type))
 
         visualizers = [
             VisualizerInstance(visualizer_handler.name, visualizer_handler.type, visualizer_handler.arg_names)
@@ -271,8 +311,8 @@ class LeapLoader:
             metric_inst = MetricInstance(metric.name, metric.arg_names)
             metrics.append(metric_inst)
 
-        return DatasetSetup(preprocess=dataset_preprocess, inputs=inputs, outputs=ground_truths, metadata=metadata,
-                            visualizers=visualizers, prediction_types=prediction_types,
+        return DatasetSetup(preprocess=dataset_preprocess, inputs=inputs, outputs=ground_truths,
+                            metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,
                             custom_losses=custom_losses, metrics=metrics)
 
     @staticmethod
@@ -287,6 +327,7 @@ class LeapLoader:
 
     @lru_cache()
     def _preprocess_result(self) -> Dict[DataStateEnum, PreprocessResponse]:
+        self.exec_script()
         preprocess = global_leap_binder.setup_container.preprocess
         # TODO: add caching of subset result
         assert preprocess is not None
@@ -325,7 +366,12 @@ class LeapLoader:
         preprocess_state = preprocess_result[state]
         for handler in global_leap_binder.setup_container.metadata:
             handler_result = handler.function(idx, preprocess_state)
-            handler_name = handler.name
-            result_agg[handler_name] = handler_result
+            if isinstance(handler_result, dict):
+                for single_metadata_name, single_metadata_result in handler_result.items():
+                    handler_name = f'{handler.name}_{single_metadata_name}'
+                    result_agg[handler_name] = single_metadata_result
+            else:
+                handler_name = handler.name
+                result_agg[handler_name] = handler_result
 
         return result_agg

code_loader-0.2.89/code_loader/visualizer_calculator_parallelized.py

@@ -0,0 +1,63 @@
+# mypy: ignore-errors
+from typing import Optional, List, Tuple, Dict
+from multiprocessing import Process, Queue
+
+import numpy as np
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from dataclasses import dataclass
+import tensorflow as tf
+from code_loader.leaploader import LeapLoader
+
+
+@dataclass
+class VisualizerSerializableError:
+    visualizer_id: str
+    visualizer_name: str
+    index_in_batch: int
+    exception_as_str: str
+
+
+class VisualizerCalculatorParallelized(LeapLoaderParallelizedBase):
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue, "spawn")
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      visualizers_to_process: Queue, ready_visualizations: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+
+        # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
+        # global param that visualizer is using)
+        leap_loader._preprocess_result()
+        leap_loader._preprocess_result.cache_clear()
+
+        while True:
+            index_in_batch, visualizer_id, visualizer_name, input_arg_name_to_tensor = \
+                visualizers_to_process.get(block=True)
+            try:
+                with tf.device('/cpu:0'):
+                    visualizer_result = \
+                        leap_loader.visualizer_by_name()[visualizer_name].function(**input_arg_name_to_tensor)
+            except Exception as e:
+                ready_visualizations.put(VisualizerSerializableError(
+                    visualizer_id, visualizer_name, index_in_batch, str(e)))
+                continue
+
+            ready_visualizations.put((index_in_batch, visualizer_id, visualizer_result))
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=VisualizerCalculatorParallelized._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def calculate_visualizers(self, input_arg_name_to_tensor_list: List[Tuple[int, str, str, Dict[str, np.array]]]):
+        return self.start_process_inputs(input_arg_name_to_tensor_list)

{code_loader-0.2.87 → code_loader-0.2.89}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "0.2.87"
+version = "0.2.89"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"