code-loader 0.2.84.dev40__py3-none-any.whl → 0.2.84.dev50__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_loader/leap_loader_parallelized_base.py +8 -8
- code_loader/metric_calculator_parallelized.py +35 -8
- code_loader/samples_generator_parallelized.py +3 -4
- {code_loader-0.2.84.dev40.dist-info → code_loader-0.2.84.dev50.dist-info}/METADATA +1 -2
- {code_loader-0.2.84.dev40.dist-info → code_loader-0.2.84.dev50.dist-info}/RECORD +7 -8
- code_loader/bla.py +0 -33
- {code_loader-0.2.84.dev40.dist-info → code_loader-0.2.84.dev50.dist-info}/LICENSE +0 -0
- {code_loader-0.2.84.dev40.dist-info → code_loader-0.2.84.dev50.dist-info}/WHEEL +0 -0
@@ -11,11 +11,12 @@ import psutil
|
|
11
11
|
|
12
12
|
class LeapLoaderParallelizedBase(ABC):
|
13
13
|
def __init__(self, code_path: str, code_entry_name: str,
|
14
|
-
n_workers: Optional[int] = 2, max_ready_results_in_queue: int = 128,
|
15
|
-
|
16
|
-
self.
|
17
|
-
if
|
18
|
-
self.
|
14
|
+
n_workers: Optional[int] = 2, max_ready_results_in_queue: int = 128,
|
15
|
+
multiprocessing_context: Optional[str] = None) -> None:
|
16
|
+
self.multiprocessing_context = multiprocessing
|
17
|
+
if multiprocessing_context is not None:
|
18
|
+
self.multiprocessing_context = multiprocessing.get_context(multiprocessing_context)
|
19
|
+
|
19
20
|
self.code_entry_name = code_entry_name
|
20
21
|
self.code_path = code_path
|
21
22
|
|
@@ -50,9 +51,8 @@ class LeapLoaderParallelizedBase(ABC):
|
|
50
51
|
|
51
52
|
@lru_cache()
|
52
53
|
def start(self) -> None:
|
53
|
-
|
54
|
-
self.
|
55
|
-
self._ready_processed_results = self.mp.Queue(self.max_ready_results_in_queue)
|
54
|
+
self._inputs_waiting_to_be_process = self.multiprocessing_context.Queue(5000)
|
55
|
+
self._ready_processed_results = self.multiprocessing_context.Queue(self.max_ready_results_in_queue)
|
56
56
|
|
57
57
|
self._run_and_warm_first_process()
|
58
58
|
n_workers = self.n_workers
|
@@ -1,21 +1,48 @@
|
|
1
1
|
# mypy: ignore-errors
|
2
|
-
import multiprocessing
|
3
2
|
from typing import Optional, List, Tuple, Dict
|
4
|
-
from multiprocessing import Process
|
5
|
-
|
3
|
+
from multiprocessing import Process, Queue
|
4
|
+
from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
|
5
|
+
import traceback
|
6
|
+
from dataclasses import dataclass
|
6
7
|
import tensorflow as tf
|
8
|
+
from code_loader.leaploader import LeapLoader
|
9
|
+
|
10
|
+
|
11
|
+
@dataclass
|
12
|
+
class MetricSerializableError:
|
13
|
+
metric_id: str
|
14
|
+
metric_name: str
|
15
|
+
leap_script_trace: str
|
16
|
+
exception_as_str: str
|
7
17
|
|
8
|
-
from code_loader.bla import _process_func
|
9
|
-
from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
|
10
18
|
|
11
19
|
class MetricCalculatorParallelized(LeapLoaderParallelizedBase):
|
12
20
|
def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
|
13
21
|
max_samples_in_queue: int = 128) -> None:
|
14
|
-
super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue)
|
22
|
+
super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue, "spawn")
|
23
|
+
|
24
|
+
@staticmethod
|
25
|
+
def _process_func(code_path: str, code_entry_name: str,
|
26
|
+
metrics_to_process: Queue, ready_samples: Queue) -> None:
|
27
|
+
import os
|
28
|
+
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
29
|
+
|
30
|
+
leap_loader = LeapLoader(code_path, code_entry_name)
|
31
|
+
while True:
|
32
|
+
metric_id, metric_name, input_arg_name_to_tensor = metrics_to_process.get(block=True)
|
33
|
+
try:
|
34
|
+
with tf.device('/cpu:0'):
|
35
|
+
metric_result = leap_loader.metric_by_name()[metric_name].function(**input_arg_name_to_tensor)
|
36
|
+
except Exception as e:
|
37
|
+
leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
|
38
|
+
ready_samples.put(MetricSerializableError(metric_id, metric_name, leap_script_trace, str(e)))
|
39
|
+
continue
|
40
|
+
|
41
|
+
ready_samples.put((metric_id, metric_result))
|
15
42
|
|
16
43
|
def _create_and_start_process(self) -> Process:
|
17
|
-
process = self.
|
18
|
-
target=_process_func,
|
44
|
+
process = self.multiprocessing_context.Process(
|
45
|
+
target=MetricCalculatorParallelized._process_func,
|
19
46
|
args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
|
20
47
|
self._ready_processed_results))
|
21
48
|
process.daemon = True
|
@@ -2,9 +2,8 @@
|
|
2
2
|
import traceback
|
3
3
|
from dataclasses import dataclass
|
4
4
|
|
5
|
-
from typing import List, Tuple, Optional
|
5
|
+
from typing import List, Tuple, Optional
|
6
6
|
|
7
|
-
import multiprocessing
|
8
7
|
from multiprocessing import Process, Queue
|
9
8
|
|
10
9
|
from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
|
@@ -23,10 +22,10 @@ class SampleSerializableError:
|
|
23
22
|
class SamplesGeneratorParallelized(LeapLoaderParallelizedBase):
|
24
23
|
def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
|
25
24
|
max_samples_in_queue: int = 128) -> None:
|
26
|
-
super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue
|
25
|
+
super().__init__(code_path, code_entry_name, n_workers, max_samples_in_queue)
|
27
26
|
|
28
27
|
def _create_and_start_process(self) -> Process:
|
29
|
-
process = self.
|
28
|
+
process = self.multiprocessing_context.Process(
|
30
29
|
target=SamplesGeneratorParallelized._process_func,
|
31
30
|
args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
|
32
31
|
self._ready_processed_results))
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: code-loader
|
3
|
-
Version: 0.2.84.
|
3
|
+
Version: 0.2.84.dev50
|
4
4
|
Summary:
|
5
5
|
Home-page: https://github.com/tensorleap/code-loader
|
6
6
|
License: MIT
|
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.8
|
13
13
|
Classifier: Programming Language :: Python :: 3.9
|
14
14
|
Classifier: Programming Language :: Python :: 3.10
|
15
|
-
Requires-Dist: multiprocess (>=0.70.15,<0.71.0)
|
16
15
|
Requires-Dist: numpy (>=1.22.3,<2.0.0)
|
17
16
|
Requires-Dist: psutil (>=5.9.5,<6.0.0)
|
18
17
|
Requires-Dist: tensorflow (>=2.11.0,<3.0.0) ; platform_machine == "x86_64"
|
@@ -1,6 +1,5 @@
|
|
1
1
|
LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
|
2
2
|
code_loader/__init__.py,sha256=ncXAeQmuP80Ml9wgSfusgLn_i6So8j1Hj7e3v4qqDdo,116
|
3
|
-
code_loader/bla.py,sha256=HUE8MQjKa25g-yTW2V-xYoY58ivtOXCaExk0eFHaPac,1135
|
4
3
|
code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
4
|
code_loader/contract/datasetclasses.py,sha256=qVk2sgAlDtkEqHq4UUOUk3JojOWQG0AX3ZBTUY09d14,4268
|
6
5
|
code_loader/contract/enums.py,sha256=KDIeNl79e8rCp4ybJat7U03j50McrcBTWUxko5SXrug,1481
|
@@ -21,16 +20,16 @@ code_loader/helpers/instancesegmentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
|
|
21
20
|
code_loader/helpers/instancesegmentation/utils.py,sha256=bl4nlg7mV8CZyPl8grGAE5603TFVM7PTZyM3BRMbXBg,930
|
22
21
|
code_loader/leap_binder/__init__.py,sha256=Oe7DNj6RmgX-tMruOrI7GYvf3SyAnzOxugF2e117_Z8,93
|
23
22
|
code_loader/leap_binder/leapbinder.py,sha256=WjC-UA5xeY9UZtMqJQOxjidlFcWURC9mmy7OrQy1IY0,6692
|
24
|
-
code_loader/leap_loader_parallelized_base.py,sha256=
|
23
|
+
code_loader/leap_loader_parallelized_base.py,sha256=_HNbjRK42wRM1V252ETVzibHW8bofFK8-Kq-It5fRks,4758
|
25
24
|
code_loader/leaploader.py,sha256=MnQeNOwNgkQItCWFiWT6xBxAsqjNbcTOPk8HIP8v19o,15370
|
26
|
-
code_loader/metric_calculator_parallelized.py,sha256=
|
25
|
+
code_loader/metric_calculator_parallelized.py,sha256=X1FHFlfPorjFVNbms19s_yItkKFja0b97wUJHNvvlxs,2233
|
27
26
|
code_loader/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
28
27
|
code_loader/metrics/default_metrics.py,sha256=xtvTskR7LC5pyKSlFcvlyBpFY9TSXNGVAi5taEPSI60,6727
|
29
|
-
code_loader/samples_generator_parallelized.py,sha256=
|
28
|
+
code_loader/samples_generator_parallelized.py,sha256=p0hQ6tKL3GI9nlDKa1GDFD1Gf3SqNRdJB640EIPwIh4,2307
|
30
29
|
code_loader/utils.py,sha256=WUcM97OuxrhfLCRPoH9EbXrxajNpYgX1CTMc3_PXtYU,1736
|
31
30
|
code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
32
31
|
code_loader/visualizers/default_visualizers.py,sha256=HqWx2qfTrroGl2n8Fpmr_4X-rk7tE2oGapjO3gzz4WY,2226
|
33
|
-
code_loader-0.2.84.
|
34
|
-
code_loader-0.2.84.
|
35
|
-
code_loader-0.2.84.
|
36
|
-
code_loader-0.2.84.
|
32
|
+
code_loader-0.2.84.dev50.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
|
33
|
+
code_loader-0.2.84.dev50.dist-info/METADATA,sha256=ABQxqPt_lK02sGPF6HUyVvKzoc450mGA3Mg6iMVuMl8,953
|
34
|
+
code_loader-0.2.84.dev50.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
|
35
|
+
code_loader-0.2.84.dev50.dist-info/RECORD,,
|
code_loader/bla.py
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
# mypy: ignore-errors
|
2
|
-
import traceback
|
3
|
-
from dataclasses import dataclass
|
4
|
-
from multiprocess import Queue
|
5
|
-
import tensorflow as tf
|
6
|
-
|
7
|
-
from code_loader.leaploader import LeapLoader
|
8
|
-
|
9
|
-
|
10
|
-
@dataclass
|
11
|
-
class MetricSerializableError:
|
12
|
-
metric_id: str
|
13
|
-
metric_name: str
|
14
|
-
leap_script_trace: str
|
15
|
-
exception_as_str: str
|
16
|
-
|
17
|
-
|
18
|
-
def _process_func(code_path: str, code_entry_name: str,
|
19
|
-
metrics_to_process: Queue, ready_samples: Queue) -> None:
|
20
|
-
# import os
|
21
|
-
# os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
|
22
|
-
leap_loader = LeapLoader(code_path, code_entry_name)
|
23
|
-
while True:
|
24
|
-
metric_id, metric_name, input_arg_name_to_tensor = metrics_to_process.get(block=True)
|
25
|
-
try:
|
26
|
-
with tf.device('/cpu:0'):
|
27
|
-
metric_result = leap_loader.metric_by_name()[metric_name].function(**input_arg_name_to_tensor)
|
28
|
-
except Exception as e:
|
29
|
-
leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
|
30
|
-
ready_samples.put(MetricSerializableError(metric_id, metric_name, leap_script_trace, str(e)))
|
31
|
-
continue
|
32
|
-
|
33
|
-
ready_samples.put((metric_id, metric_result))
|
File without changes
|
File without changes
|