code-loader 0.2.84.dev40__py3-none-any.whl → 0.2.84.dev50__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,11 +11,12 @@ import psutil
11
11
 
12
12
  class LeapLoaderParallelizedBase(ABC):
13
13
  def __init__(self, code_path: str, code_entry_name: str,
14
- n_workers: Optional[int] = 2, max_ready_results_in_queue: int = 128, bb = "spawn") -> None:
15
- self.bb = bb
16
- self.mp = multiprocessing
17
- if self.bb is not None:
18
- self.mp = multiprocessing.get_context(self.bb)
14
+ n_workers: Optional[int] = 2, max_ready_results_in_queue: int = 128,
15
+ multiprocessing_context: Optional[str] = None) -> None:
16
+ self.multiprocessing_context = multiprocessing
17
+ if multiprocessing_context is not None:
18
+ self.multiprocessing_context = multiprocessing.get_context(multiprocessing_context)
19
+
19
20
  self.code_entry_name = code_entry_name
20
21
  self.code_path = code_path
21
22
 
@@ -50,9 +51,8 @@ class LeapLoaderParallelizedBase(ABC):
50
51
 
51
52
  @lru_cache()
52
53
  def start(self) -> None:
53
- # multiprocessing.set_start_method(self.bb, force=True)
54
- self._inputs_waiting_to_be_process = self.mp.Queue(5000)
55
- self._ready_processed_results = self.mp.Queue(self.max_ready_results_in_queue)
54
+ self._inputs_waiting_to_be_process = self.multiprocessing_context.Queue(5000)
55
+ self._ready_processed_results = self.multiprocessing_context.Queue(self.max_ready_results_in_queue)
56
56
 
57
57
  self._run_and_warm_first_process()
58
58
  n_workers = self.n_workers
@@ -1,21 +1,48 @@
1
1
  # mypy: ignore-errors
2
- import multiprocessing
3
2
  from typing import Optional, List, Tuple, Dict
4
- from multiprocessing import Process
5
-
3
+ from multiprocessing import Process, Queue
4
+ from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
5
+ import traceback
6
+ from dataclasses import dataclass
6
7
  import tensorflow as tf
8
+ from code_loader.leaploader import LeapLoader
9
+
10
+
11
+ @dataclass
12
+ class MetricSerializableError:
13
+ metric_id: str
14
+ metric_name: str
15
+ leap_script_trace: str
16
+ exception_as_str: str
7
17
 
8
- from code_loader.bla import _process_func
9
- from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
10
18
 
11
19
  class MetricCalculatorParallelized(LeapLoaderParallelizedBase):
12
20
  def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
13
21
  max_samples_in_queue: int = 128) -> None:
14
- super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue)
22
+ super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue, "spawn")
23
+
24
+ @staticmethod
25
+ def _process_func(code_path: str, code_entry_name: str,
26
+ metrics_to_process: Queue, ready_samples: Queue) -> None:
27
+ import os
28
+ os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
29
+
30
+ leap_loader = LeapLoader(code_path, code_entry_name)
31
+ while True:
32
+ metric_id, metric_name, input_arg_name_to_tensor = metrics_to_process.get(block=True)
33
+ try:
34
+ with tf.device('/cpu:0'):
35
+ metric_result = leap_loader.metric_by_name()[metric_name].function(**input_arg_name_to_tensor)
36
+ except Exception as e:
37
+ leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
38
+ ready_samples.put(MetricSerializableError(metric_id, metric_name, leap_script_trace, str(e)))
39
+ continue
40
+
41
+ ready_samples.put((metric_id, metric_result))
15
42
 
16
43
  def _create_and_start_process(self) -> Process:
17
- process = self.mp.Process(
18
- target=_process_func,
44
+ process = self.multiprocessing_context.Process(
45
+ target=MetricCalculatorParallelized._process_func,
19
46
  args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
20
47
  self._ready_processed_results))
21
48
  process.daemon = True
@@ -2,9 +2,8 @@
2
2
  import traceback
3
3
  from dataclasses import dataclass
4
4
 
5
- from typing import List, Tuple, Optional, Dict
5
+ from typing import List, Tuple, Optional
6
6
 
7
- import multiprocessing
8
7
  from multiprocessing import Process, Queue
9
8
 
10
9
  from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
@@ -23,10 +22,10 @@ class SampleSerializableError:
23
22
  class SamplesGeneratorParallelized(LeapLoaderParallelizedBase):
24
23
  def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
25
24
  max_samples_in_queue: int = 128) -> None:
26
- super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue, None)
25
+ super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue)
27
26
 
28
27
  def _create_and_start_process(self) -> Process:
29
- process = self.mp.Process(
28
+ process = self.multiprocessing_context.Process(
30
29
  target=SamplesGeneratorParallelized._process_func,
31
30
  args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
32
31
  self._ready_processed_results))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: code-loader
3
- Version: 0.2.84.dev40
3
+ Version: 0.2.84.dev50
4
4
  Summary:
5
5
  Home-page: https://github.com/tensorleap/code-loader
6
6
  License: MIT
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3
12
12
  Classifier: Programming Language :: Python :: 3.8
13
13
  Classifier: Programming Language :: Python :: 3.9
14
14
  Classifier: Programming Language :: Python :: 3.10
15
- Requires-Dist: multiprocess (>=0.70.15,<0.71.0)
16
15
  Requires-Dist: numpy (>=1.22.3,<2.0.0)
17
16
  Requires-Dist: psutil (>=5.9.5,<6.0.0)
18
17
  Requires-Dist: tensorflow (>=2.11.0,<3.0.0) ; platform_machine == "x86_64"
@@ -1,6 +1,5 @@
1
1
  LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
2
2
  code_loader/__init__.py,sha256=ncXAeQmuP80Ml9wgSfusgLn_i6So8j1Hj7e3v4qqDdo,116
3
- code_loader/bla.py,sha256=HUE8MQjKa25g-yTW2V-xYoY58ivtOXCaExk0eFHaPac,1135
4
3
  code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
4
  code_loader/contract/datasetclasses.py,sha256=qVk2sgAlDtkEqHq4UUOUk3JojOWQG0AX3ZBTUY09d14,4268
6
5
  code_loader/contract/enums.py,sha256=KDIeNl79e8rCp4ybJat7U03j50McrcBTWUxko5SXrug,1481
@@ -21,16 +20,16 @@ code_loader/helpers/instancesegmentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
21
20
  code_loader/helpers/instancesegmentation/utils.py,sha256=bl4nlg7mV8CZyPl8grGAE5603TFVM7PTZyM3BRMbXBg,930
22
21
  code_loader/leap_binder/__init__.py,sha256=Oe7DNj6RmgX-tMruOrI7GYvf3SyAnzOxugF2e117_Z8,93
23
22
  code_loader/leap_binder/leapbinder.py,sha256=WjC-UA5xeY9UZtMqJQOxjidlFcWURC9mmy7OrQy1IY0,6692
24
- code_loader/leap_loader_parallelized_base.py,sha256=kkddNrIFQKBWciT6YdIV8qoJ6977UQzNDYLZaIbgDWg,4680
23
+ code_loader/leap_loader_parallelized_base.py,sha256=_HNbjRK42wRM1V252ETVzibHW8bofFK8-Kq-It5fRks,4758
25
24
  code_loader/leaploader.py,sha256=MnQeNOwNgkQItCWFiWT6xBxAsqjNbcTOPk8HIP8v19o,15370
26
- code_loader/metric_calculator_parallelized.py,sha256=HNuoHLNbfzcWDU0Lx3WMIrstLmwJHkgM6dV06IeI7nc,1101
25
+ code_loader/metric_calculator_parallelized.py,sha256=X1FHFlfPorjFVNbms19s_yItkKFja0b97wUJHNvvlxs,2233
27
26
  code_loader/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
28
27
  code_loader/metrics/default_metrics.py,sha256=xtvTskR7LC5pyKSlFcvlyBpFY9TSXNGVAi5taEPSI60,6727
29
- code_loader/samples_generator_parallelized.py,sha256=Zpz-Ko3dkz9R73z7bSVQL42J8otTUwBnjq02fZp8lBg,2321
28
+ code_loader/samples_generator_parallelized.py,sha256=p0hQ6tKL3GI9nlDKa1GDFD1Gf3SqNRdJB640EIPwIh4,2307
30
29
  code_loader/utils.py,sha256=WUcM97OuxrhfLCRPoH9EbXrxajNpYgX1CTMc3_PXtYU,1736
31
30
  code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
31
  code_loader/visualizers/default_visualizers.py,sha256=HqWx2qfTrroGl2n8Fpmr_4X-rk7tE2oGapjO3gzz4WY,2226
33
- code_loader-0.2.84.dev40.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
34
- code_loader-0.2.84.dev40.dist-info/METADATA,sha256=exjW-HtU9XR0o61-Z52yFekthUcSeE5Hf_DjCbh0COg,1001
35
- code_loader-0.2.84.dev40.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
36
- code_loader-0.2.84.dev40.dist-info/RECORD,,
32
+ code_loader-0.2.84.dev50.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
33
+ code_loader-0.2.84.dev50.dist-info/METADATA,sha256=ABQxqPt_lK02sGPF6HUyVvKzoc450mGA3Mg6iMVuMl8,953
34
+ code_loader-0.2.84.dev50.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
35
+ code_loader-0.2.84.dev50.dist-info/RECORD,,
code_loader/bla.py DELETED
@@ -1,33 +0,0 @@
1
- # mypy: ignore-errors
2
- import traceback
3
- from dataclasses import dataclass
4
- from multiprocess import Queue
5
- import tensorflow as tf
6
-
7
- from code_loader.leaploader import LeapLoader
8
-
9
-
10
- @dataclass
11
- class MetricSerializableError:
12
- metric_id: str
13
- metric_name: str
14
- leap_script_trace: str
15
- exception_as_str: str
16
-
17
-
18
- def _process_func(code_path: str, code_entry_name: str,
19
- metrics_to_process: Queue, ready_samples: Queue) -> None:
20
- # import os
21
- # os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
22
- leap_loader = LeapLoader(code_path, code_entry_name)
23
- while True:
24
- metric_id, metric_name, input_arg_name_to_tensor = metrics_to_process.get(block=True)
25
- try:
26
- with tf.device('/cpu:0'):
27
- metric_result = leap_loader.metric_by_name()[metric_name].function(**input_arg_name_to_tensor)
28
- except Exception as e:
29
- leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
30
- ready_samples.put(MetricSerializableError(metric_id, metric_name, leap_script_trace, str(e)))
31
- continue
32
-
33
- ready_samples.put((metric_id, metric_result))