code-loader 1.0.58__py3-none-any.whl → 1.0.59__py3-none-any.whl

code_loader/code_inegration_processes_manager.py

@@ -0,0 +1,83 @@
+# mypy: ignore-errors
+import traceback
+from dataclasses import dataclass
+
+from typing import List, Tuple, Optional
+
+from multiprocessing import Process, Queue
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from code_loader.leaploader import LeapLoader
+from code_loader.contract.enums import DataStateEnum
+from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
+from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
+
+
+@dataclass
+class SampleSerializableError:
+    state: DataStateEnum
+    index: int
+    leap_script_trace: str
+    exception_as_str: str
+
+
+class CodeIntegrationProcessesManager:
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
+        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=CodeIntegrationProcessesManager._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def _run_and_warm_first_process(self):
+        process = self._create_and_start_process()
+        self.processes = [process]
+
+        # needed in order to make sure the preprocess func runs once in nonparallel
+        self._start_process_inputs([(DataStateEnum.training, 0)])
+        self._get_next_ready_processed_result()
+
+    def _operation_decider(self):
+        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
+            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'metric'
+
+        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
+            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'dataset'
+
+
+
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      samples_to_process: Queue, ready_samples: Queue,
+                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+        while True:
+
+            # decide on sample or metric to process
+            state, idx = samples_to_process.get(block=True)
+            leap_loader._preprocess_result()
+            try:
+                sample = leap_loader.get_sample(state, idx)
+            except Exception as e:
+                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
+                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
+                continue
+
+            ready_samples.put(sample)
+
+    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
+        return self.start_process_inputs(sample_identities)
+

code_loader/inner_leap_binder/leapbinder_decorators.py

@@ -357,9 +357,9 @@ def tensorleap_custom_loss(name: str):
                 (f'tensorleap_custom_loss validation failed: '
                  f'The return type should be a numpy array or a tensorflow tensor. Got {type(result)}.')
 
-        def inner(sample_id, preprocess_response):
-            _validate_input_args(sample_id, preprocess_response)
-            result = user_function(sample_id, preprocess_response)
+        def inner(*args, **kwargs):
+            _validate_input_args(*args, **kwargs)
+            result = user_function(*args, **kwargs)
             _validate_result(result)
             return result
 

code_loader/leaploader.py

@@ -19,7 +19,6 @@ from code_loader.contract.exceptions import DatasetScriptException
 from code_loader.contract.responsedataclasses import DatasetIntegParseResult, DatasetTestResultPayload, \
     DatasetPreprocess, DatasetSetup, DatasetInputInstance, DatasetOutputInstance, DatasetMetadataInstance, \
     VisualizerInstance, PredictionTypeInstance, ModelSetup, CustomLayerInstance, MetricInstance, CustomLossInstance
-from code_loader.dualstream import DualStream
 from code_loader.inner_leap_binder import global_leap_binder
 from code_loader.utils import get_root_exception_file_and_line_number
 
@@ -124,9 +123,8 @@ class LeapLoader:
         test_payloads: List[DatasetTestResultPayload] = []
         setup_response = None
         general_error = None
-        stdout_stream = io.StringIO()
-        dual_stream = DualStream(sys.stdout, stdout_stream)
-        with redirect_stdout(dual_stream):
+        stdout_steam = io.StringIO()
+        with redirect_stdout(stdout_steam):
             try:
                 self.exec_script()
                 preprocess_test_payload = self._check_preprocess()
@@ -144,7 +142,7 @@ class LeapLoader:
                 general_error = f"Something went wrong. {repr(e.__cause__)} in file {file_name}, line_number:  {line_number}\nStacktrace:\n{stacktrace}"
                 is_valid = False
 
-        print_log = dual_stream.stream2.getvalue()
+        print_log = stdout_steam.getvalue()
         is_valid_for_model = bool(global_leap_binder.setup_container.custom_layers)
         model_setup = self.get_model_setup_response()
 

{code_loader-1.0.58.dist-info → code_loader-1.0.59.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.58
+Version: 1.0.59
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.58.dist-info → code_loader-1.0.59.dist-info}/RECORD

@@ -1,12 +1,12 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
+code_loader/code_inegration_processes_manager.py,sha256=XslWOPeNQk4RAFJ_f3tP5Oe3EgcIR7BE7Y8r9Ty73-o,3261
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/contract/datasetclasses.py,sha256=lFS7_weizsjzx4_tYwYGrrRUj1sgIl010h9FON4brb8,6670
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
 code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F4RiyQ,51
 code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
 code_loader/contract/visualizer_classes.py,sha256=iIa_O2rKvPTwN5ILCTZvRpsGYiiFABKdwQwfIXGigDo,11928
-code_loader/dualstream.py,sha256=mlpwTEiFSHFMg2xd9Gd3pv5n0HLWdRAxA-kbtdhRt4M,807
 code_loader/experiment_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/experiment_api/api.py,sha256=a7wh6Hhe7IaVxu46eV2soSz-yxnmXG3ipU1BBtsEAaQ,2493
 code_loader/experiment_api/cli_config_utils.py,sha256=n6JMyNrquxql3KKxHhAP8jAzezlRT-PV2KWI95kKsm0,1140
@@ -19,12 +19,12 @@ code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZ
 code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
 code_loader/inner_leap_binder/leapbinder.py,sha256=35hyesDdmjOD9wdrTLyayb-vm9aDfmEbMA0c4EQR1LA,25090
-code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=b7CTPrXpSi-rmrD9gO9vu_usSDH-SOeg25xTgzznWIk,19749
-code_loader/leaploader.py,sha256=w3r7QsFSXvqfdZoHV1t_61G3IVdLNAeNMWZcAeTfu0A,19604
+code_loader/inner_leap_binder/leapbinder_decorators.py,sha256=uuM_ht9HZ1GH2IabKeGQ_x9NmD3poK_h1Gt0NruwJuY,19704
+code_loader/leaploader.py,sha256=POUgD6x1GH_iF_eDGz-VLX4DsIl2kddufKVDdrA_K-U,19491
 code_loader/utils.py,sha256=aw2i_fqW_ADjLB66FWZd9DfpCQ7mPdMyauROC5Nd51I,2197
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=VoqO9FN84yXyMjRjHjUTOt2GdTkJRMbHbXJ1cJkREkk,2230
-code_loader-1.0.58.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.58.dist-info/METADATA,sha256=-e9EK4Yc5QdPFaIUQcKW6fos5KnkmTkTDY-4zM3oN8A,888
-code_loader-1.0.58.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.58.dist-info/RECORD,,
+code_loader-1.0.59.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.59.dist-info/METADATA,sha256=F3b6BUv0E3jFu6ciMsIu0EzSPQMlnP1aKKG7GGSTu0I,888
+code_loader-1.0.59.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.59.dist-info/RECORD,,

code_loader/dualstream.py

@@ -1,32 +0,0 @@
-from io import StringIO
-from typing import IO
-
-
-class DualStream(StringIO):
-    def __init__(self, stream1: IO[str], stream2: StringIO):
-        super().__init__()
-        self.stream1 = stream1  # Usually sys.stdout
-        self.stream2 = stream2  # The StringIO stream
-
-    def write(self, s: str) -> int:
-        # Write to both streams and return the length of the written string
-        self.stream1.write(s)
-        self.stream2.write(s)
-        return len(s)
-
-    def flush(self) -> None:
-        self.stream1.flush()
-        self.stream2.flush()
-
-    def close(self) -> None:
-        # Do not close sys.stdout
-        self.stream2.close()
-
-    def readable(self) -> bool:
-        return False
-
-    def writable(self) -> bool:
-        return True
-
-    def seekable(self) -> bool:
-        return False