code-loader 0.2.88__py3-none-any.whl → 0.2.89__py3-none-any.whl

code_loader/leap_loader_parallelized_base.py

@@ -32,13 +32,13 @@ class LeapLoaderParallelizedBase(ABC):
         self._generate_inputs_thread: Optional[Thread] = None
         self._should_stop_thread = False
 
-    def _calculate_n_workers_by_hardware(self) -> int:
+    def _calculate_n_workers_bpyproject.tomly_hardware(self) -> int:
         p = psutil.Process(self.processes[0].pid)
         memory_usage_in_bytes = p.memory_info().rss
         total_memory_in_bytes = psutil.virtual_memory().total
 
         n_workers = min(int(multiprocessing.cpu_count()),
-                        int(total_memory_in_bytes * 0.7 / memory_usage_in_bytes))
+                        int(total_memory_in_bytes * 0.5 / memory_usage_in_bytes))
         n_workers = max(n_workers, 1)
         return n_workers
 

code_loader/leaploader.py

@@ -327,6 +327,7 @@ class LeapLoader:
 
     @lru_cache()
     def _preprocess_result(self) -> Dict[DataStateEnum, PreprocessResponse]:
+        self.exec_script()
         preprocess = global_leap_binder.setup_container.preprocess
         # TODO: add caching of subset result
         assert preprocess is not None

code_loader/visualizer_calculator_parallelized.py

@@ -0,0 +1,63 @@
+# mypy: ignore-errors
+from typing import Optional, List, Tuple, Dict
+from multiprocessing import Process, Queue
+
+import numpy as np
+
+from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
+from dataclasses import dataclass
+import tensorflow as tf
+from code_loader.leaploader import LeapLoader
+
+
+@dataclass
+class VisualizerSerializableError:
+    visualizer_id: str
+    visualizer_name: str
+    index_in_batch: int
+    exception_as_str: str
+
+
+class VisualizerCalculatorParallelized(LeapLoaderParallelizedBase):
+    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
+                 max_samples_in_queue: int = 128) -> None:
+        super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue, "spawn")
+
+    @staticmethod
+    def _process_func(code_path: str, code_entry_name: str,
+                      visualizers_to_process: Queue, ready_visualizations: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+        leap_loader = LeapLoader(code_path, code_entry_name)
+
+        # running preprocessing to sync preprocessing in main thread (can be valuable when preprocess is filling a
+        # global param that visualizer is using)
+        leap_loader._preprocess_result()
+        leap_loader._preprocess_result.cache_clear()
+
+        while True:
+            index_in_batch, visualizer_id, visualizer_name, input_arg_name_to_tensor = \
+                visualizers_to_process.get(block=True)
+            try:
+                with tf.device('/cpu:0'):
+                    visualizer_result = \
+                        leap_loader.visualizer_by_name()[visualizer_name].function(**input_arg_name_to_tensor)
+            except Exception as e:
+                ready_visualizations.put(VisualizerSerializableError(
+                    visualizer_id, visualizer_name, index_in_batch, str(e)))
+                continue
+
+            ready_visualizations.put((index_in_batch, visualizer_id, visualizer_result))
+
+    def _create_and_start_process(self) -> Process:
+        process = self.multiprocessing_context.Process(
+            target=VisualizerCalculatorParallelized._process_func,
+            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
+                  self._ready_processed_results))
+        process.daemon = True
+        process.start()
+        return process
+
+    def calculate_visualizers(self, input_arg_name_to_tensor_list: List[Tuple[int, str, str, Dict[str, np.array]]]):
+        return self.start_process_inputs(input_arg_name_to_tensor_list)

{code_loader-0.2.88.dist-info → code_loader-0.2.89.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 0.2.88
+Version: 0.2.89
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-0.2.88.dist-info → code_loader-0.2.89.dist-info}/RECORD

@@ -20,16 +20,17 @@ code_loader/helpers/instancesegmentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 code_loader/helpers/instancesegmentation/utils.py,sha256=bl4nlg7mV8CZyPl8grGAE5603TFVM7PTZyM3BRMbXBg,930
 code_loader/leap_binder/__init__.py,sha256=Oe7DNj6RmgX-tMruOrI7GYvf3SyAnzOxugF2e117_Z8,93
 code_loader/leap_binder/leapbinder.py,sha256=-Ny2YewT1jg36Q5AXK8F2Fmv34zxhbcXZp807jIcHws,6293
-code_loader/leap_loader_parallelized_base.py,sha256=_HNbjRK42wRM1V252ETVzibHW8bofFK8-Kq-It5fRks,4758
-code_loader/leaploader.py,sha256=iHz5Hq7phFmjxsH1wr6d_Fajwg2JFEBwYzDoXdR46qo,18496
+code_loader/leap_loader_parallelized_base.py,sha256=4G7Ch3DmgArqmXAKyLCG5JJCgJ6V9LgueBDVKSyts0w,4772
+code_loader/leaploader.py,sha256=wb-eiGw6p-CML8w3noESEjJLjXJ_AiHxfAlGxSBtjYA,18523
 code_loader/metric_calculator_parallelized.py,sha256=X1FHFlfPorjFVNbms19s_yItkKFja0b97wUJHNvvlxs,2233
 code_loader/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/metrics/default_metrics.py,sha256=xtvTskR7LC5pyKSlFcvlyBpFY9TSXNGVAi5taEPSI60,6727
 code_loader/samples_generator_parallelized.py,sha256=_t0lmf11y1Tas4bdndOSk2b2QMjBQt9N_voTg-tcTuQ,2376
 code_loader/utils.py,sha256=WUcM97OuxrhfLCRPoH9EbXrxajNpYgX1CTMc3_PXtYU,1736
+code_loader/visualizer_calculator_parallelized.py,sha256=dKAixQ7-TIcG2sEkvbDbxKyuZP0WJP5oLO2bqN2mWRY,2597
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=HqWx2qfTrroGl2n8Fpmr_4X-rk7tE2oGapjO3gzz4WY,2226
-code_loader-0.2.88.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-0.2.88.dist-info/METADATA,sha256=RVdQb6WajrbescPNX6lposmhOIa9dTkWLDOuPL0NS10,950
-code_loader-0.2.88.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-code_loader-0.2.88.dist-info/RECORD,,
+code_loader-0.2.89.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-0.2.89.dist-info/METADATA,sha256=5L_HuArCRwuKYwHVcGXqcD_xanKChIFKcWtNdL-7tgU,950
+code_loader-0.2.89.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+code_loader-0.2.89.dist-info/RECORD,,