code-loader 1.0.60.dev2__py3-none-any.whl → 1.0.61.dev1__py3-none-any.whl

code_loader/code_inegration_processes_manager.py

@@ -0,0 +1,83 @@
+# mypy: ignore-errors
+import traceback
+from dataclasses import dataclass
+
+from typing import List, Tuple, Optional
+
+from multiprocessing import Process, Queue
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from code_loader.leaploader import LeapLoader
+from code_loader.contract.enums import DataStateEnum
+from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
+from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
+
+
+@dataclass
+class SampleSerializableError:
+    state: DataStateEnum
+    index: int
+    leap_script_trace: str
+    exception_as_str: str
+
+
+class CodeIntegrationProcessesManager:
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
+        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=CodeIntegrationProcessesManager._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def _run_and_warm_first_process(self):
+        process = self._create_and_start_process()
+        self.processes = [process]
+
+        # needed in order to make sure the preprocess func runs once in nonparallel
+        self._start_process_inputs([(DataStateEnum.training, 0)])
+        self._get_next_ready_processed_result()
+
+    def _operation_decider(self):
+        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
+            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'metric'
+
+        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
+            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'dataset'
+
+
+
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      samples_to_process: Queue, ready_samples: Queue,
+                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+        while True:
+
+            # decide on sample or metric to process
+            state, idx = samples_to_process.get(block=True)
+            leap_loader._preprocess_result()
+            try:
+                sample = leap_loader.get_sample(state, idx)
+            except Exception as e:
+                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
+                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
+                continue
+
+            ready_samples.put(sample)
+
+    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
+        return self.start_process_inputs(sample_identities)
+

code_loader/contract/datasetclasses.py

@@ -154,7 +154,7 @@ class DatasetBaseHandler:
 @dataclass
 class InputHandler(DatasetBaseHandler):
     shape: Optional[List[int]] = None
-    channel_dim: Optional[int] = -1
+
 
 @dataclass
 class GroundTruthHandler(DatasetBaseHandler):

code_loader/contract/responsedataclasses.py

@@ -20,7 +20,6 @@ class DatasetBaseSectionInstance:
 @dataclass
 class DatasetInputInstance(DatasetBaseSectionInstance):
     shape: List[int]
-    channel_dim: Optional[int] = -1
 
 
 @dataclass

code_loader/inner_leap_binder/leapbinder.py

@@ -181,14 +181,13 @@ class LeapBinder:
         """
         self.setup_container.unlabeled_data_preprocess = UnlabeledDataPreprocessHandler(function)
 
-    def set_input(self, function: SectionCallableInterface, name: str, channel_dim: int = -1) -> None:
+    def set_input(self, function: SectionCallableInterface, name: str) -> None:
         """
         Set the input handler function.
 
         Args:
         function (SectionCallableInterface): The input handler function.
         name (str): The name of the input section.
-        channel_dim (int): The dimension of the channels axis
 
         Example:
             def input_encoder(subset: PreprocessResponse, index: int) -> np.ndarray:
@@ -198,10 +197,10 @@ class LeapBinder:
                 img = normalize(img)
                 return img
 
-            leap_binder.set_input(input_encoder, name='input_encoder', channel_dim=-1)
+            leap_binder.set_input(input_encoder, name='input_encoder')
         """
         function = to_numpy_return_wrapper(function)
-        self.setup_container.inputs.append(InputHandler(name, function, channel_dim=channel_dim))
+        self.setup_container.inputs.append(InputHandler(name, function))
 
         self._encoder_names.append(name)
 

code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -245,16 +245,14 @@ def tensorleap_unlabeled_preprocess():
     return decorating_function
 
 
-def tensorleap_input_encoder(name: str, channel_dim=-1):
+def tensorleap_input_encoder(name: str):
     def decorating_function(user_function: SectionCallableInterface):
         for input_handler in leap_binder.setup_container.inputs:
             if input_handler.name == name:
                 raise Exception(f'Input with name {name} already exists. '
                                 f'Please choose another')
-        if channel_dim < 0 and channel_dim != -1:
-            raise Exception(f"Channel dim for input {name} is expected to be either -1 or positive")
 
-        leap_binder.set_input(user_function, name, channel_dim=channel_dim)
+        leap_binder.set_input(user_function, name)
 
         def _validate_input_args(sample_id: Union[int, str], preprocess_response: PreprocessResponse):
             assert isinstance(sample_id, (int, str)), \
@@ -275,8 +273,6 @@ def tensorleap_input_encoder(name: str, channel_dim=-1):
             assert result.dtype == np.float32, \
                 (f'tensorleap_input_encoder validation failed: '
                  f'The return type should be a numpy array of type float32. Got {result.dtype}.')
-            assert channel_dim - 1 <= len(result.shape), (f'tensorleap_input_encoder validation failed: '
-                 f'The channel_dim ({channel_dim}) should be <= to the rank of the resulting input rank ({len(result.shape)}).')
 
         def inner(sample_id, preprocess_response):
             _validate_input_args(sample_id, preprocess_response)

code_loader/leaploader.py

@@ -2,7 +2,6 @@
 import importlib.util
 import io
 import sys
-import time
 from contextlib import redirect_stdout
 from functools import lru_cache
 from pathlib import Path
@@ -20,13 +19,13 @@ from code_loader.contract.responsedataclasses import DatasetIntegParseResult, Da
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance
 from code_loader.inner_leap_binder import global_leap_binder
+from code_loader.leaploaderbase import LeapLoaderBase
 from code_loader.utils import get_root_exception_file_and_line_number
 
 
-class LeapLoader:
+class LeapLoader(LeapLoaderBase):
     def __init__(self, code_path: str, code_entry_name: str):
-        self.code_entry_name = code_entry_name
-        self.code_path = code_path
+        super().__init__(code_path, code_entry_name)
 
         self._preprocess_result_cached = None
 
@@ -202,6 +201,10 @@ class LeapLoader:
 
     def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
                        ) -> LeapData:
+        # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
+        # global param that visualizer is using)
+        self._preprocess_result()
+
         return self.visualizer_by_name()[visualizer_name].function(**input_tensors_by_arg_name)
 
     def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
@@ -212,8 +215,7 @@ class LeapLoader:
             return list(input_tensors_by_arg_name.values())[0]
         return heatmap_function(**input_tensors_by_arg_name)
 
-    @staticmethod
-    def get_dataset_setup_response(handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
+    def get_dataset_setup_response(self, handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
         setup = global_leap_binder.setup_container
         assert setup.preprocess is not None
 
@@ -231,7 +233,7 @@ class LeapLoader:
         for inp in setup.inputs:
             if inp.shape is None:
                 raise Exception(f"cant calculate shape for input, input name:{inp.name}")
-            inputs.append(DatasetInputInstance(name=inp.name, shape=inp.shape, channel_dim=inp.channel_dim))
+            inputs.append(DatasetInputInstance(name=inp.name, shape=inp.shape))
 
         ground_truths = []
         for gt in setup.ground_truths:
@@ -283,8 +285,7 @@ class LeapLoader:
                             metadata=metadata_instances, visualizers=visualizers, prediction_types=prediction_types,
                             custom_losses=custom_losses, metrics=metrics)
 
-    @staticmethod
-    def get_model_setup_response() -> ModelSetup:
+    def get_model_setup_response(self) -> ModelSetup:
         setup = global_leap_binder.setup_container
         custom_layer_instances = [
             CustomLayerInstance(custom_layer_handler.name, custom_layer_handler.init_arg_names,

code_loader/leaploaderbase.py

@@ -0,0 +1,77 @@
+# mypy: ignore-errors
+
+from abc import abstractmethod
+
+from typing import Dict, List, Union, Type
+
+import numpy as np
+import numpy.typing as npt
+
+from code_loader.contract.datasetclasses import DatasetSample, VisualizerHandler, LeapData, CustomLossHandler, \
+    PredictionTypeHandler, CustomLayerHandler
+from code_loader.contract.enums import DataStateEnum
+from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
+    DatasetSetup, ModelSetup
+
+
+class LeapLoaderBase:
+    def __init__(self, code_path: str, code_entry_name: str):
+        self.code_entry_name = code_entry_name
+        self.code_path = code_path
+
+        self._preprocess_result_cached = None
+
+    @abstractmethod
+    def metric_by_name(self):
+        pass
+
+    @abstractmethod
+    def visualizer_by_name(self) -> Dict[str, VisualizerHandler]:
+        pass
+
+    @abstractmethod
+    def custom_loss_by_name(self) -> Dict[str, CustomLossHandler]:
+        pass
+
+    @abstractmethod
+    def custom_layers(self) -> Dict[str, CustomLayerHandler]:
+        pass
+
+    @abstractmethod
+    def prediction_type_by_name(self) -> Dict[str, PredictionTypeHandler]:
+        pass
+
+    @abstractmethod
+    def get_sample(self, state: DataStateEnum, sample_id: Union[int, str]) -> DatasetSample:
+        pass
+
+    @abstractmethod
+    def check_dataset(self) -> DatasetIntegParseResult:
+        pass
+
+    @abstractmethod
+    def run_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]],
+                       ) -> LeapData:
+        pass
+
+    @abstractmethod
+    def run_heatmap_visualizer(self, visualizer_name: str, input_tensors_by_arg_name: Dict[str, npt.NDArray[np.float32]]
+                               ) -> npt.NDArray[np.float32]:
+        pass
+
+    @abstractmethod
+    def get_dataset_setup_response(self, handlers_test_payloads: List[DatasetTestResultPayload]) -> DatasetSetup:
+        pass
+
+    @abstractmethod
+    def get_model_setup_response(self) -> ModelSetup:
+        pass
+
+    @abstractmethod
+    def get_preprocess_sample_ids(
+            self, update_unlabeled_preprocess=False) -> Dict[DataStateEnum, Union[List[int], List[str]]]:
+        pass
+
+    @abstractmethod
+    def get_sample_id_type(self) -> Type:
+        pass

{code_loader-1.0.60.dev2.dist-info → code_loader-1.0.61.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.60.dev2
+Version: 1.0.61.dev1
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.60.dev2.dist-info → code_loader-1.0.61.dev1.dist-info}/RECORD

@@ -1,10 +1,11 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
+code_loader/code_inegration_processes_manager.py,sha256=XslWOPeNQk4RAFJ_f3tP5Oe3EgcIR7BE7Y8r9Ty73-o,3261
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/contract/datasetclasses.py,sha256=cd6fRDC4XLrJa7PcrzoTPIKtGwFZq09DGwiSC5BSKvk,6705
+code_loader/contract/datasetclasses.py,sha256=lFS7_weizsjzx4_tYwYGrrRUj1sgIl010h9FON4brb8,6670
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
-code_loader/contract/responsedataclasses.py,sha256=RSx9m_R3LawhK5o1nAcO3hfp2F9oJYtxZr_bpP3bTmw,4005
+code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
 code_loader/contract/visualizer_classes.py,sha256=iIa_O2rKvPTwN5ILCTZvRpsGYiiFABKdwQwfIXGigDo,11928
 code_loader/experiment_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/experiment_api/api.py,sha256=a7wh6Hhe7IaVxu46eV2soSz-yxnmXG3ipU1BBtsEAaQ,2493
@@ -17,13 +18,14 @@ code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1o
 code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=LVzpynjISO-a774flzGt1yAQPsSYNE8B5V58Hacs7bQ,25216
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=6yHYZRFb5wv1dEOV6r75kIN4-cxXtFDbgHL9ciQQ_x4,20131
-code_loader/leaploader.py,sha256=Tpf6A25hYuo4D0umGL3BHNYJhmz_NIwvFveQgAlsSOo,19534
+code_loader/inner_leap_binder/leapbinder.py,sha256=35hyesDdmjOD9wdrTLyayb-vm9aDfmEbMA0c4EQR1LA,25090
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=uuM_ht9HZ1GH2IabKeGQ_x9NmD3poK_h1Gt0NruwJuY,19704
+code_loader/leaploader.py,sha256=KC_6oso5pbOHZ56sUTcV6qdFzEbIJ8MdEtKu-nDrQfE,19707
+code_loader/leaploaderbase.py,sha256=vVgy2AJXuB0Sf6ry49r_PxvcPOwwz9RXD3VCtxqTFC0,2284
 code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=VoqO9FN84yXyMjRjHjUTOt2GdTkJRMbHbXJ1cJkREkk,2230
-code_loader-1.0.60.dev2.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.60.dev2.dist-info/METADATA,sha256=Qd_qx-0HBEh0g-Yimq2xX-aQ0CJwpNuPSUDNVCmKcwA,893
-code_loader-1.0.60.dev2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.60.dev2.dist-info/RECORD,,
+code_loader-1.0.61.dev1.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.61.dev1.dist-info/METADATA,sha256=iym33XwG43oBOA1gFvLKu2MMh3LHPVfSH930mqUs8Ds,893
+code_loader-1.0.61.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.61.dev1.dist-info/RECORD,,