code-loader 1.0.11__py3-none-any.whl → 1.0.12__py3-none-any.whl

code_loader/code_inegration_processes_manager.py

@@ -0,0 +1,83 @@
+# mypy: ignore-errors
+import traceback
+from dataclasses import dataclass
+
+from typing import List, Tuple, Optional
+
+from multiprocessing import Process, Queue
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from code_loader.leaploader import LeapLoader
+from code_loader.contract.enums import DataStateEnum
+from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
+from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
+
+
+@dataclass
+class SampleSerializableError:
+    state: DataStateEnum
+    index: int
+    leap_script_trace: str
+    exception_as_str: str
+
+
+class CodeIntegrationProcessesManager:
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
+        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=CodeIntegrationProcessesManager._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def _run_and_warm_first_process(self):
+        process = self._create_and_start_process()
+        self.processes = [process]
+
+        # needed in order to make sure the preprocess func runs once in nonparallel
+        self._start_process_inputs([(DataStateEnum.training, 0)])
+        self._get_next_ready_processed_result()
+
+    def _operation_decider(self):
+        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
+            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'metric'
+
+        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
+            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
+            return 'dataset'
+
+
+
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      samples_to_process: Queue, ready_samples: Queue,
+                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+        while True:
+
+            # decide on sample or metric to process
+            state, idx = samples_to_process.get(block=True)
+            leap_loader._preprocess_result()
+            try:
+                sample = leap_loader.get_sample(state, idx)
+            except Exception as e:
+                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
+                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
+                continue
+
+            ready_samples.put(sample)
+
+    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
+        return self.start_process_inputs(sample_identities)
+

code_loader/leap_loader_parallelized_base.py

@@ -133,3 +133,8 @@ class LeapLoaderParallelizedBase(ABC):
 
     def __del__(self) -> None:
         self.release()
+
+    def check_processes(self) -> None:
+        for process in self.processes:
+            if not process.is_alive():
+                raise Exception(f'Sub process unexpected failure. Exit code: {process.exitcode}')

{code_loader-1.0.11.dist-info → code_loader-1.0.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.11
+Version: 1.0.12
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.11.dist-info → code_loader-1.0.12.dist-info}/RECORD

@@ -1,5 +1,6 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=V3DEXSN6Ie6PlGeSAbzjp9ufRj0XPJLpD7pDLLYxk6M,122
+code_loader/code_inegration_processes_manager.py,sha256=XslWOPeNQk4RAFJ_f3tP5Oe3EgcIR7BE7Y8r9Ty73-o,3261
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/contract/datasetclasses.py,sha256=gh0J7pFcD659fP6BZ4Dr5esGR6ewZklASjagp_5so5M,4527
 code_loader/contract/enums.py,sha256=KDIeNl79e8rCp4ybJat7U03j50McrcBTWUxko5SXrug,1481
@@ -8,7 +9,7 @@ code_loader/contract/responsedataclasses.py,sha256=thIxKSx3mwoKHYICULGaeh2-S56L1
 code_loader/contract/visualizer_classes.py,sha256=1FjVO744J_EMuJfHWXGdvSz6vl3Vu7iS3CDfs8MzEEQ,5138
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
 code_loader/inner_leap_binder/leapbinder.py,sha256=HaHzek7ClKKkJ8I8MW92-Mv6xPRoRE-pmUVh22J8uWg,12883
-code_loader/leap_loader_parallelized_base.py,sha256=NqJL9rmNNSEOMHpecEzsHdkN0EGn4ASRqNkSyQ3v56Q,4758
+code_loader/leap_loader_parallelized_base.py,sha256=KuxlDECh4dimY7g8rNqYiqneurAtLmsAJ1hi2PvGo7o,4974
 code_loader/leaploader.py,sha256=_RZ5SL88holzEVZbxoVEJo6wlYIEc5toIH592letfRM,15424
 code_loader/metric_calculator_parallelized.py,sha256=aCOw4NCrqTUQkaLrglYxWBK9XGWZ1M9cySti2HaS4dA,2181
 code_loader/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -18,7 +19,7 @@ code_loader/utils.py,sha256=WUcM97OuxrhfLCRPoH9EbXrxajNpYgX1CTMc3_PXtYU,1736
 code_loader/visualizer_calculator_parallelized.py,sha256=V3G9BOka5b_QIsyBCeK1NYofkbD5Wej01eGXP3dceK8,2523
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=HqWx2qfTrroGl2n8Fpmr_4X-rk7tE2oGapjO3gzz4WY,2226
-code_loader-1.0.11.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.11.dist-info/METADATA,sha256=XxxgWCRIL1ObV5uUWV-i450aZSle4dtc9Se5aoPacWw,956
-code_loader-1.0.11.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-code_loader-1.0.11.dist-info/RECORD,,
+code_loader-1.0.12.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.12.dist-info/METADATA,sha256=1gc_yOGHTqroxlHtMzxfiiXlsp9rmd4fcNJlYLxfUPQ,956
+code_loader-1.0.12.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+code_loader-1.0.12.dist-info/RECORD,,

{code_loader-1.0.11.dist-info → code_loader-1.0.12.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.7.0
+Generator: poetry-core 1.6.1
 Root-Is-Purelib: true
 Tag: py3-none-any