code-loader 0.2.84.dev32__py3-none-any.whl → 0.2.84.dev50__py3-none-any.whl

code_loader/leap_loader_parallelized_base.py

@@ -13,8 +13,9 @@ class LeapLoaderParallelizedBase(ABC):
     def __init__(self, code_path: str, code_entry_name: str,
                  n_workers: Optional[int] = 2, max_ready_results_in_queue: int = 128,
                  multiprocessing_context: Optional[str] = None) -> None:
-        self.bb = multiprocessing_context
-        self.multiprocessing_context = multiprocessing.get_context(multiprocessing_context)
+        self.multiprocessing_context = multiprocessing
+        if multiprocessing_context is not None:
+            self.multiprocessing_context = multiprocessing.get_context(multiprocessing_context)
 
         self.code_entry_name = code_entry_name
         self.code_path = code_path
@@ -36,7 +37,7 @@ class LeapLoaderParallelizedBase(ABC):
         memory_usage_in_bytes = p.memory_info().rss
         total_memory_in_bytes = psutil.virtual_memory().total
 
-        n_workers = min(int(self.multiprocessing_context.cpu_count()),
+        n_workers = min(int(multiprocessing.cpu_count()),
                         int(total_memory_in_bytes * 0.7 / memory_usage_in_bytes))
         n_workers = max(n_workers, 1)
         return n_workers
@@ -50,10 +51,8 @@ class LeapLoaderParallelizedBase(ABC):
 
     @lru_cache()
     def start(self) -> None:
-        if self.bb is not None:
-            multiprocessing.set_start_method("spawn", force=True)
-        self._inputs_waiting_to_be_process = multiprocessing.Queue(5000)
-        self._ready_processed_results = multiprocessing.Queue(self.max_ready_results_in_queue)
+        self._inputs_waiting_to_be_process = self.multiprocessing_context.Queue(5000)
+        self._ready_processed_results = self.multiprocessing_context.Queue(self.max_ready_results_in_queue)
 
         self._run_and_warm_first_process()
         n_workers = self.n_workers

code_loader/metric_calculator_parallelized.py

@@ -1,5 +1,4 @@
 # mypy: ignore-errors
-import multiprocessing
 from typing import Optional, List, Tuple, Dict
 from multiprocessing import Process, Queue
 from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
@@ -25,6 +24,9 @@ class MetricCalculatorParallelized(LeapLoaderParallelizedBase):
     @staticmethod
     def _process_func(code_path: str, code_entry_name: str,
                       metrics_to_process: Queue, ready_samples: Queue) -> None:
+        import os
+        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
         leap_loader = LeapLoader(code_path, code_entry_name)
         while True:
             metric_id, metric_name, input_arg_name_to_tensor = metrics_to_process.get(block=True)
@@ -39,7 +41,7 @@ class MetricCalculatorParallelized(LeapLoaderParallelizedBase):
             ready_samples.put((metric_id, metric_result))
 
     def _create_and_start_process(self) -> Process:
-        process = multiprocessing.Process(
+        process = self.multiprocessing_context.Process(
             target=MetricCalculatorParallelized._process_func,
             args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
                   self._ready_processed_results))

code_loader/samples_generator_parallelized.py

@@ -1,5 +1,4 @@
 # mypy: ignore-errors
-import multiprocessing
 import traceback
 from dataclasses import dataclass
 
@@ -26,7 +25,7 @@ class SamplesGeneratorParallelized(LeapLoaderParallelizedBase):
         super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue)
 
     def _create_and_start_process(self) -> Process:
-        process = multiprocessing.Process(
+        process = self.multiprocessing_context.Process(
             target=SamplesGeneratorParallelized._process_func,
             args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
                   self._ready_processed_results))

{code_loader-0.2.84.dev32.dist-info → code_loader-0.2.84.dev50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 0.2.84.dev32
+Version: 0.2.84.dev50
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-0.2.84.dev32.dist-info → code_loader-0.2.84.dev50.dist-info}/RECORD

@@ -20,16 +20,16 @@ code_loader/helpers/instancesegmentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 code_loader/helpers/instancesegmentation/utils.py,sha256=bl4nlg7mV8CZyPl8grGAE5603TFVM7PTZyM3BRMbXBg,930
 code_loader/leap_binder/__init__.py,sha256=Oe7DNj6RmgX-tMruOrI7GYvf3SyAnzOxugF2e117_Z8,93
 code_loader/leap_binder/leapbinder.py,sha256=WjC-UA5xeY9UZtMqJQOxjidlFcWURC9mmy7OrQy1IY0,6692
-code_loader/leap_loader_parallelized_base.py,sha256=v4bpdCmu4U12MjMwfMqGo9JkwKi81hipv3kQZrvr21g,4778
+code_loader/leap_loader_parallelized_base.py,sha256=_HNbjRK42wRM1V252ETVzibHW8bofFK8-Kq-It5fRks,4758
 code_loader/leaploader.py,sha256=MnQeNOwNgkQItCWFiWT6xBxAsqjNbcTOPk8HIP8v19o,15370
-code_loader/metric_calculator_parallelized.py,sha256=nKUXYDnSgF4U84CAqNKYnskl4CgOogpO0L8dUxJC2Fo,2174
+code_loader/metric_calculator_parallelized.py,sha256=X1FHFlfPorjFVNbms19s_yItkKFja0b97wUJHNvvlxs,2233
 code_loader/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/metrics/default_metrics.py,sha256=xtvTskR7LC5pyKSlFcvlyBpFY9TSXNGVAi5taEPSI60,6727
-code_loader/samples_generator_parallelized.py,sha256=lz6JomnwTQTk4Ci6K78l-h0yW9mL4gK1-yJiLLZSilM,2317
+code_loader/samples_generator_parallelized.py,sha256=p0hQ6tKL3GI9nlDKa1GDFD1Gf3SqNRdJB640EIPwIh4,2307
 code_loader/utils.py,sha256=WUcM97OuxrhfLCRPoH9EbXrxajNpYgX1CTMc3_PXtYU,1736
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=HqWx2qfTrroGl2n8Fpmr_4X-rk7tE2oGapjO3gzz4WY,2226
-code_loader-0.2.84.dev32.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-0.2.84.dev32.dist-info/METADATA,sha256=m6ucdstaO2lBpGNxBTE4rVHeZUfRzmjRI8kttRQAdEM,953
-code_loader-0.2.84.dev32.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-code_loader-0.2.84.dev32.dist-info/RECORD,,
+code_loader-0.2.84.dev50.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-0.2.84.dev50.dist-info/METADATA,sha256=ABQxqPt_lK02sGPF6HUyVvKzoc450mGA3Mg6iMVuMl8,953
+code_loader-0.2.84.dev50.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+code_loader-0.2.84.dev50.dist-info/RECORD,,