code-loader 1.0.64a0__tar.gz → 1.0.64.dev1__tar.gz

{code_loader-1.0.64a0 → code_loader-1.0.64.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.64a0
+Version: 1.0.64.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

code_loader-1.0.64.dev1/code_loader/code_inegration_processes_manager.py

@@ -0,0 +1,83 @@
+# mypy: ignore-errors
+import traceback
+from dataclasses import dataclass
+
+from typing import List, Tuple, Optional
+
+from multiprocessing import Process, Queue
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from code_loader.leaploader import LeapLoader
+from code_loader.contract.enums import DataStateEnum
+from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
+from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
+
+
+@dataclass
+class SampleSerializableError:
+    state: DataStateEnum
+    index: int
+    leap_script_trace: str
+    exception_as_str: str
+
+
+class CodeIntegrationProcessesManager:
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
+        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=CodeIntegrationProcessesManager._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def _run_and_warm_first_process(self):
+        process = self._create_and_start_process()
+        self.processes = [process]
+
+        # needed in order to make sure the preprocess func runs once in nonparallel
+        self._start_process_inputs([(DataStateEnum.training, 0)])
+        self._get_next_ready_processed_result()
+
+    def _operation_decider(self):
+        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
+            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'metric'
+
+        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
+            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'dataset'
+
+
+
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      samples_to_process: Queue, ready_samples: Queue,
+                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+        while True:
+
+            # decide on sample or metric to process
+            state, idx = samples_to_process.get(block=True)
+            leap_loader._preprocess_result()
+            try:
+                sample = leap_loader.get_sample(state, idx)
+            except Exception as e:
+                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
+                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
+                continue
+
+            ready_samples.put(sample)
+
+    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
+        return self.start_process_inputs(sample_identities)
+

{code_loader-1.0.64a0 → code_loader-1.0.64.dev1}/code_loader/contract/datasetclasses.py

@@ -117,31 +117,46 @@ MetricCallableReturnType = Union[Any, List[List[ConfusionMatrixElement]]]
 
 
 @dataclass
-class CustomLossHandler:
+class CustomLossHandlerData:
     name: str
-    function: CustomCallableInterface
     arg_names: List[str]
 
 
 @dataclass
-class MetricHandler:
+class CustomLossHandler:
+    custom_loss_handler_data: CustomLossHandlerData
+    function: CustomCallableInterface
+
+
+@dataclass
+class MetricHandlerData:
     name: str
-    function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
     arg_names: List[str]
     direction: Optional[MetricDirection] = MetricDirection.Downward
 
 
+@dataclass
+class MetricHandler:
+    metric_handler_data: MetricHandlerData
+    function: Union[CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs]
+
+
 @dataclass
 class RawInputsForHeatmap:
     raw_input_by_vizualizer_arg_name: Dict[str, npt.NDArray[np.float32]]
 
 
 @dataclass
-class VisualizerHandler:
+class VisualizerHandlerData:
     name: str
-    function: VisualizerCallableInterface
     type: LeapDataType
     arg_names: List[str]
+
+
+@dataclass
+class VisualizerHandler:
+    visualizer_handler_data: VisualizerHandlerData
+    function: VisualizerCallableInterface
     heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None
 
 
@@ -181,7 +196,6 @@ class CustomLayerHandler:
     init_arg_names: List[str]
     call_arg_names: List[str]
     use_custom_latent_space: bool = False
-    build: bool = True
 
 
 @dataclass

{code_loader-1.0.64a0 → code_loader-1.0.64.dev1}/code_loader/inner_leap_binder/leapbinder.py

@@ -9,7 +9,8 @@ from code_loader.contract.datasetclasses import SectionCallableInterface, InputH
     PreprocessHandler, VisualizerCallableInterface, CustomLossHandler, CustomCallableInterface, PredictionTypeHandler, \
     MetadataSectionCallableInterface, UnlabeledDataPreprocessHandler, CustomLayerHandler, MetricHandler, \
     CustomCallableInterfaceMultiArgs, ConfusionMatrixCallableInterfaceMultiArgs, LeapData, \
-    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, RawInputsForHeatmap
+    CustomMultipleReturnCallableInterfaceMultiArgs, DatasetBaseHandler, custom_latent_space_attribute, \
+    RawInputsForHeatmap, VisualizerHandlerData, MetricHandlerData, CustomLossHandlerData
 from code_loader.contract.enums import LeapDataType, DataStateEnum, DataStateType, MetricDirection
 from code_loader.contract.responsedataclasses import DatasetTestResultPayload
 from code_loader.contract.visualizer_classes import map_leap_data_type_to_visualizer_class
@@ -127,7 +128,7 @@ class LeapBinder:
                     f'should be {expected_return_type}')
 
         self.setup_container.visualizers.append(
-            VisualizerHandler(name, function, visualizer_type, arg_names, heatmap_visualizer))
+            VisualizerHandler(VisualizerHandlerData(name, visualizer_type, arg_names), function, heatmap_visualizer))
         self._visualizer_names.append(name)
 
     def set_preprocess(self, function: Callable[[], List[PreprocessResponse]]) -> None:
@@ -226,7 +227,7 @@ class LeapBinder:
             leap_binder.add_custom_loss(custom_loss_function, name='custom_loss')
         """
         arg_names = inspect.getfullargspec(function)[0]
-        self.setup_container.custom_loss_handlers.append(CustomLossHandler(name, function, arg_names))
+        self.setup_container.custom_loss_handlers.append(CustomLossHandler(CustomLossHandlerData(name, arg_names), function))
 
     def add_custom_metric(self,
                           function: Union[CustomCallableInterfaceMultiArgs,
@@ -252,7 +253,7 @@ class LeapBinder:
             leap_binder.add_custom_metric(custom_metric_function, name='custom_metric', direction=MetricDirection.Downward)
         """
         arg_names = inspect.getfullargspec(function)[0]
-        self.setup_container.metrics.append(MetricHandler(name, function, arg_names, direction))
+        self.setup_container.metrics.append(MetricHandler(MetricHandlerData(name, arg_names, direction), function))
 
     def add_prediction(self, name: str, labels: List[str], channel_dim: int = -1) -> None:
         """
@@ -339,8 +340,7 @@ class LeapBinder:
         self.setup_container.metadata.append(MetadataHandler(name, function))
 
     def set_custom_layer(self, custom_layer: Type[Any], name: str, inspect_layer: bool = False,
-                         kernel_index: Optional[int] = None, use_custom_latent_space: bool = False,
-                         build=True) -> None:
+                         kernel_index: Optional[int] = None, use_custom_latent_space: bool = False) -> None:
         """
         Set a custom layer for the model.
 
@@ -370,8 +370,7 @@ class LeapBinder:
         init_args = inspect.getfullargspec(custom_layer.__init__)[0][1:]
         call_args = inspect.getfullargspec(custom_layer.call)[0][1:]
         self.setup_container.custom_layers[name] = CustomLayerHandler(name, custom_layer, init_args, call_args,
-                                                                      use_custom_latent_space=use_custom_latent_space,
-                                                                      build=build)
+                                                                      use_custom_latent_space=use_custom_latent_space)
 
     def check_preprocess(self, preprocess_result: Dict[DataStateEnum, PreprocessResponse]) -> None:
         preprocess_handler = self.setup_container.preprocess

{code_loader-1.0.64a0 → code_loader-1.0.64.dev1}/code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -22,7 +22,7 @@ def tensorleap_custom_metric(name: str, direction: Optional[MetricDirection] = M
             ConfusionMatrixCallableInterfaceMultiArgs]
     ):
         for metric_handler in leap_binder.setup_container.metrics:
-            if metric_handler.name == name:
+            if metric_handler.metric_handler_data.name == name:
                 raise Exception(f'Metric with name {name} already exists. '
                                 f'Please choose another')
 
@@ -94,7 +94,7 @@ def tensorleap_custom_visualizer(name: str, visualizer_type: LeapDataType,
                                  heatmap_function: Optional[Callable[..., npt.NDArray[np.float32]]] = None):
     def decorating_function(user_function: VisualizerCallableInterface):
         for viz_handler in leap_binder.setup_container.visualizers:
-            if viz_handler.name == name:
+            if viz_handler.visualizer_handler_data.name == name:
                 raise Exception(f'Visualizer with name {name} already exists. '
                                 f'Please choose another')
 
@@ -334,7 +334,7 @@ def tensorleap_gt_encoder(name: str):
 def tensorleap_custom_loss(name: str):
     def decorating_function(user_function: CustomCallableInterface):
         for loss_handler in leap_binder.setup_container.custom_loss_handlers:
-            if loss_handler.name == name:
+            if loss_handler.custom_loss_handler_data.name == name:
                 raise Exception(f'Custom loss with name {name} already exists. '
                                 f'Please choose another')
 

{code_loader-1.0.64a0 → code_loader-1.0.64.dev1}/code_loader/leaploader.py

@@ -1,32 +1,33 @@
 # mypy: ignore-errors
 import importlib.util
+import inspect
 import io
 import sys
-import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
-from typing import Dict, List, Iterable, Union, Any, Type
+from typing import Dict, List, Iterable, Union, Any, Type, Optional
 
 import numpy as np
 import numpy.typing as npt
 
 from code_loader.contract.datasetclasses import DatasetSample, DatasetBaseHandler, GroundTruthHandler, \
-    PreprocessResponse, VisualizerHandler, LeapData, CustomLossHandler, \
-    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler
+    PreprocessResponse, VisualizerHandler, LeapData, \
+    PredictionTypeHandler, MetadataHandler, CustomLayerHandler, MetricHandler, VisualizerHandlerData, MetricHandlerData, \
+    MetricCallableReturnType, CustomLossHandlerData, CustomLossHandler, RawInputsForHeatmap
 from code_loader.contract.enums import DataStateEnum, TestingSectionEnum, DataStateType, DatasetMetadataType
 from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance
 from code_loader.inner_leap_binder import global_leap_binder
+from code_loader.leaploaderbase import LeapLoaderBase
 from code_loader.utils import get_root_exception_file_and_line_number
 
 
-class LeapLoader:
+class LeapLoader(LeapLoaderBase):
     def __init__(self, code_path: str, code_entry_name: str):
-        self.code_entry_name = code_entry_name
-        self.code_path = code_path
+        super().__init__(code_path, code_entry_name)
 
         self._preprocess_result_cached = None
 
@@ -66,29 +67,56 @@ class LeapLoader:
         spec.loader.exec_module(file)
 
     @lru_cache()
-    def metric_by_name(self) -> Dict[str, MetricHandler]:
+    def metric_by_name(self) -> Dict[str, MetricHandlerData]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
-            metric_handler.name: metric_handler
+            metric_handler.metric_handler_data.name: metric_handler.metric_handler_data
             for metric_handler in setup.metrics
         }
 
     @lru_cache()
-    def visualizer_by_name(self) -> Dict[str, VisualizerHandler]:
+    def _metric_handler_by_name(self) -> Dict[str, MetricHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
-            visualizer_handler.name: visualizer_handler
+            metric_handler.metric_handler_data.name: metric_handler
+            for metric_handler in setup.metrics
+        }
+
+    @lru_cache()
+    def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            visualizer_handler.visualizer_handler_data.name: visualizer_handler.visualizer_handler_data
             for visualizer_handler in setup.visualizers
         }
 
     @lru_cache()
-    def custom_loss_by_name(self) -> Dict[str, CustomLossHandler]:
+    def _visualizer_handler_by_name(self) -> Dict[str, VisualizerHandler]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            visualizer_handler.visualizer_handler_data.name: visualizer_handler
+            for visualizer_handler in setup.visualizers
+        }
+
+    @lru_cache()
+    def custom_loss_by_name(self) -> Dict[str, CustomLossHandlerData]:
+        self.exec_script()
+        setup = global_leap_binder.setup_container
+        return {
+            custom_loss_handler.custom_loss_handler_data.name: custom_loss_handler.custom_loss_handler_data
+            for custom_loss_handler in setup.custom_loss_handlers
+        }
+
+    @lru_cache()
+    def _custom_loss_handler_by_name(self) -> Dict[str, CustomLossHandler]:
         self.exec_script()
         setup = global_leap_binder.setup_container
         return {
-            custom_loss_handler.name: custom_loss_handler
+            custom_loss_handler.custom_loss_handler_data.name: custom_loss_handler
             for custom_loss_handler in setup.custom_loss_handlers
         }
 
@@ -200,20 +228,41 @@ class LeapLoader:
         all_dataset_base_handlers.extend(global_leap_binder.setup_container.metadata)
         return all_dataset_base_handlers
 
-    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
-                       ) -> LeapData:
-        return self.visualizer_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
+    def run_metric(self, metric_name: str,
+                   input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
+        self._preprocess_result()
+        return self._metric_handler_by_name()[metric_name].function(**input_tensors_by_arg_name)
+
+    def run_custom_loss(self, custom_loss_name: str,
+                        input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
+        return self._custom_loss_handler_by_name()[custom_loss_name].function(**input_tensors_by_arg_name)
+
+    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
+        # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
+        # global param that visualizer is using)
+        self._preprocess_result()
+
+        return self._visualizer_handler_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
 
     def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
                                ) -> npt.NDArray[np.float32]:
-        heatmap_function = self.visualizer_by_name()[visualizer_name].heatmap_function
+        heatmap_function = self._visualizer_handler_by_name()[visualizer_name].heatmap_function
         if heatmap_function is None:
             assert len(input_tensors_by_arg_name) == 1
             return list(input_tensors_by_arg_name.values())[0]
         return heatmap_function(**input_tensors_by_arg_name)
 
-    @staticmethod
-    def get_dataset_setup_response(handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
+    def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
+        heatmap_function = self._visualizer_handler_by_name()[visualizer_name].heatmap_function
+        if heatmap_function is None:
+            return None
+
+        for arg_name, arg_type in inspect.getfullargspec(heatmap_function).annotations.items():
+            if arg_type == RawInputsForHeatmap:
+                return arg_name
+        return None
+
+    def get_dataset_setup_response(self, handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
         setup = global_leap_binder.setup_container
         assert setup.preprocess is not None
 
@@ -262,10 +311,13 @@ class LeapLoader:
                                                               type=dataset_metadata_type))
 
         visualizers = [
-            VisualizerInstance(visualizer_handler.name, visualizer_handler.type, visualizer_handler.arg_names)
+            VisualizerInstance(
+                visualizer_handler.visualizer_handler_data.name, visualizer_handler.visualizer_handler_data.type,
+                visualizer_handler.visualizer_handler_data.arg_names)
             for visualizer_handler in setup.visualizers]
 
-        custom_losses = [CustomLossInstance(custom_loss.name, custom_loss.arg_names)
+        custom_losses = [CustomLossInstance(custom_loss.custom_loss_handler_data.name,
+                                            custom_loss.custom_loss_handler_data.arg_names)
                          for custom_loss in setup.custom_loss_handlers]
 
         prediction_types = []
@@ -276,15 +328,14 @@ class LeapLoader:
 
         metrics = []
         for metric in setup.metrics:
-            metric_inst = MetricInstance(metric.name, metric.arg_names)
+            metric_inst = MetricInstance(metric.metric_handler_data.name, metric.metric_handler_data.arg_names)
             metrics.append(metric_inst)
 
         return DatasetSetup(preprocess=dataset_preprocess, inputs=inputs, outputs=ground_truths,
                             metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,
                             custom_losses=custom_losses, metrics=metrics)
 
-    @staticmethod
-    def get_model_setup_response() -> ModelSetup:
+    def get_model_setup_response(self) -> ModelSetup:
         setup = global_leap_binder.setup_container
         custom_layer_instances = [
             CustomLayerInstance(custom_layer_handler.name, custom_layer_handler.init_arg_names,
@@ -389,3 +440,4 @@ class LeapLoader:
 
         return id_type
 
+

code_loader-1.0.64.dev1/code_loader/leaploaderbase.py

@@ -0,0 +1,95 @@
+# mypy: ignore-errors
+
+from abc import abstractmethod
+
+from typing import Dict, List, Union, Type, Optional
+
+import numpy as np
+import numpy.typing as npt
+
+from code_loader.contract.datasetclasses import DatasetSample, LeapData, \
+    PredictionTypeHandler, CustomLayerHandler, VisualizerHandlerData, MetricHandlerData, MetricCallableReturnType, \
+    CustomLossHandlerData
+from code_loader.contract.enums import DataStateEnum
+from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
+    DatasetSetup, ModelSetup
+
+
+class LeapLoaderBase:
+    def __init__(self, code_path: str, code_entry_name: str):
+        self.code_entry_name = code_entry_name
+        self.code_path = code_path
+
+    @abstractmethod
+    def metric_by_name(self) -> Dict[str, MetricHandlerData]:
+        pass
+
+    @abstractmethod
+    def visualizer_by_name(self) -> Dict[str, VisualizerHandlerData]:
+        pass
+
+    @abstractmethod
+    def custom_loss_by_name(self) -> Dict[str, CustomLossHandlerData]:
+        pass
+
+    @abstractmethod
+    def custom_layers(self) -> Dict[str, CustomLayerHandler]:
+        pass
+
+    @abstractmethod
+    def prediction_type_by_name(self) -> Dict[str, PredictionTypeHandler]:
+        pass
+
+    @abstractmethod
+    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
+        pass
+
+    @abstractmethod
+    def check_dataset(self) -> DatasetIntegParseResult:
+        pass
+
+    @abstractmethod
+    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> LeapData:
+        pass
+
+    @abstractmethod
+    def run_metric(self, metric_name: str,
+                   input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]) -> MetricCallableReturnType:
+        pass
+
+    @abstractmethod
+    def run_custom_loss(self, custom_loss_name: str,
+                        input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]):
+        pass
+
+    @abstractmethod
+    def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                               ) -> npt.NDArray[np.float32]:
+        pass
+
+    @abstractmethod
+    def get_dataset_setup_response(self, handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
+        pass
+
+    @abstractmethod
+    def get_model_setup_response(self) -> ModelSetup:
+        pass
+
+    @abstractmethod
+    def get_preprocess_sample_ids(
+            self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
+        pass
+
+    @abstractmethod
+    def get_sample_id_type(self) -> Type:
+        pass
+
+    @abstractmethod
+    def get_heatmap_visualizer_raw_vis_input_arg_name(self, visualizer_name: str) -> Optional[str]:
+        pass
+
+    def is_custom_latent_space(self) -> bool:
+        if not self.code_entry_name or not self.code_path:
+            return False
+        custom_layers = self.custom_layers()
+        return any(layer.use_custom_latent_space for layer in custom_layers.values())

{code_loader-1.0.64a0 → code_loader-1.0.64.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.64a0"
+version = "1.0.64.dev1"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"