labga 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- labga/__init__.py +28 -0
- labga/_evaluation_worker.py +181 -0
- labga/_resources.py +62 -0
- labga/_types.py +26 -0
- labga/_worker_pool.py +204 -0
- labga/evaluation.py +272 -0
- labga/genomes.py +411 -0
- labga/islands.py +64 -0
- labga/objective.py +169 -0
- labga/operators.py +101 -0
- labga/optimize.py +261 -0
- labga/predictors.py +89 -0
- labga/py.typed +0 -0
- labga/results.py +68 -0
- labga-0.1.0.dist-info/METADATA +228 -0
- labga-0.1.0.dist-info/RECORD +18 -0
- labga-0.1.0.dist-info/WHEEL +4 -0
- labga-0.1.0.dist-info/licenses/LICENSE +21 -0
labga/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Genetic algorithms for fixed-predictor optimization."""
|
|
2
|
+
|
|
3
|
+
from labga.evaluation import EvaluationError
|
|
4
|
+
from labga.genomes import BinaryGenome, Normal, NumericGenome
|
|
5
|
+
from labga.islands import Migration
|
|
6
|
+
from labga.objective import Objective
|
|
7
|
+
from labga.operators import Operators
|
|
8
|
+
from labga.optimize import optimize
|
|
9
|
+
from labga.predictors import Predictor, TorchPredictor
|
|
10
|
+
from labga.results import GenerationRecord, OptimizationResult, Population
|
|
11
|
+
|
|
12
|
+
__version__ = "0.1.0"
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"BinaryGenome",
|
|
16
|
+
"EvaluationError",
|
|
17
|
+
"GenerationRecord",
|
|
18
|
+
"Migration",
|
|
19
|
+
"Normal",
|
|
20
|
+
"NumericGenome",
|
|
21
|
+
"Objective",
|
|
22
|
+
"Operators",
|
|
23
|
+
"OptimizationResult",
|
|
24
|
+
"Population",
|
|
25
|
+
"Predictor",
|
|
26
|
+
"TorchPredictor",
|
|
27
|
+
"optimize",
|
|
28
|
+
]
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Model evaluation and preprocessing executed exclusively by their owners."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any, cast
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
from numpy.typing import NDArray
|
|
10
|
+
|
|
11
|
+
from labga._types import FloatArray
|
|
12
|
+
from labga.objective import Objective, Preprocess, _real_array
|
|
13
|
+
from labga.predictors import Predictor
|
|
14
|
+
|
|
15
|
+
Array = NDArray[Any]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Job:
|
|
20
|
+
"""Candidate IDs and their encoded inputs for a bounded submission."""
|
|
21
|
+
|
|
22
|
+
ids: tuple[int, ...]
|
|
23
|
+
rows: Array
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class Reply:
|
|
28
|
+
"""Row-level evidence and current adaptive capacity from one submission."""
|
|
29
|
+
|
|
30
|
+
completed: dict[int, tuple[FloatArray, float]] = field(default_factory=dict)
|
|
31
|
+
errors: dict[int, str] = field(default_factory=dict)
|
|
32
|
+
prepared: dict[int, Array] = field(default_factory=dict)
|
|
33
|
+
widths: dict[int, int] = field(default_factory=dict)
|
|
34
|
+
cap: int = 1
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def describe(exc: BaseException) -> str:
|
|
38
|
+
return f"{type(exc).__name__}: {exc}"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def prepare(job: Job, preprocess: Preprocess) -> Reply:
|
|
42
|
+
reply = Reply()
|
|
43
|
+
for index, row in zip(job.ids, job.rows, strict=True):
|
|
44
|
+
try:
|
|
45
|
+
value = _real_array(preprocess(row.copy()), "preprocess")
|
|
46
|
+
if value.ndim != 1 or not value.size or not np.isfinite(value).all():
|
|
47
|
+
raise ValueError("preprocess must return a nonempty finite numeric vector")
|
|
48
|
+
reply.prepared[index] = value.copy()
|
|
49
|
+
reply.widths[index] = len(value)
|
|
50
|
+
except Exception as exc:
|
|
51
|
+
reply.errors[index] = describe(exc)
|
|
52
|
+
return reply
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class WorkerSpec:
|
|
57
|
+
"""Only preprocessing configuration is sent to preprocessing children."""
|
|
58
|
+
|
|
59
|
+
objective: Objective | None = None
|
|
60
|
+
preprocess: Preprocess | None = None
|
|
61
|
+
cap: int = 64
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Evaluator:
|
|
65
|
+
"""Persistent predictor state shared by all requests served by one owner."""
|
|
66
|
+
|
|
67
|
+
def __init__(self, objective: Objective, cap: int) -> None:
|
|
68
|
+
self.objective = objective
|
|
69
|
+
config = objective.predictor
|
|
70
|
+
self.model = config.build() if isinstance(config, Predictor) else config
|
|
71
|
+
self.adaptive = (
|
|
72
|
+
isinstance(config, Predictor)
|
|
73
|
+
and config.device.startswith("cuda")
|
|
74
|
+
and config.batch_size == "auto"
|
|
75
|
+
)
|
|
76
|
+
self.predict_batch = getattr(self.model, "predict_batch", None)
|
|
77
|
+
self.batched = callable(self.predict_batch)
|
|
78
|
+
self.cap = cap if self.batched else 1
|
|
79
|
+
|
|
80
|
+
def _score(self, ids: tuple[int, ...], raw: Any, reply: Reply) -> None:
|
|
81
|
+
expected = (len(ids), len(np.asarray(self.objective.target)))
|
|
82
|
+
if not hasattr(raw, "shape"):
|
|
83
|
+
if isinstance(raw, (list, tuple)):
|
|
84
|
+
# Infer a common array only when this preserves real-valued peer rows.
|
|
85
|
+
# A ragged, complex, or textual row must be diagnosed independently.
|
|
86
|
+
try:
|
|
87
|
+
array = np.asarray(raw)
|
|
88
|
+
except (TypeError, ValueError):
|
|
89
|
+
pass
|
|
90
|
+
else:
|
|
91
|
+
if array.dtype.kind in "biuf":
|
|
92
|
+
raw = array
|
|
93
|
+
else:
|
|
94
|
+
raw = np.asarray(raw)
|
|
95
|
+
shape = getattr(raw, "shape", None)
|
|
96
|
+
row_count = len(raw) if shape is None or len(shape) else None
|
|
97
|
+
if row_count != len(ids):
|
|
98
|
+
raise ValueError(f"prediction shape must be {expected}; got {shape}")
|
|
99
|
+
if shape == expected and isinstance(self.objective.loss, str):
|
|
100
|
+
try:
|
|
101
|
+
batch = self.objective.score(raw, count=len(ids))
|
|
102
|
+
except ValueError:
|
|
103
|
+
# Localize invalid predictions or overflowing built-in losses.
|
|
104
|
+
# Neither model calls nor user-defined losses are repeated.
|
|
105
|
+
pass
|
|
106
|
+
else:
|
|
107
|
+
for index, pred, loss in zip(ids, batch.predictions, batch.losses, strict=True):
|
|
108
|
+
reply.completed[index] = (pred, float(loss))
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
for pos, index in enumerate(ids):
|
|
112
|
+
try:
|
|
113
|
+
row = raw[pos]
|
|
114
|
+
if not hasattr(row, "shape"):
|
|
115
|
+
row = np.asarray(row)
|
|
116
|
+
single = self.objective.score(row[None, ...], count=1)
|
|
117
|
+
reply.completed[index] = (single.predictions[0], float(single.losses[0]))
|
|
118
|
+
except Exception as exc:
|
|
119
|
+
reply.errors[index] = describe(exc)
|
|
120
|
+
|
|
121
|
+
def _inference(self, job: Job, reply: Reply) -> None:
|
|
122
|
+
if not self.batched:
|
|
123
|
+
single = cast(Callable[[Array], Any], getattr(self.model, "predict", self.model))
|
|
124
|
+
for index, row in zip(job.ids, job.rows, strict=True):
|
|
125
|
+
try:
|
|
126
|
+
raw = single(row.copy())
|
|
127
|
+
if not hasattr(raw, "shape"):
|
|
128
|
+
raw = np.asarray(raw)
|
|
129
|
+
self._score((index,), raw[None, ...], reply)
|
|
130
|
+
except Exception as exc:
|
|
131
|
+
reply.errors[index] = describe(exc)
|
|
132
|
+
return
|
|
133
|
+
|
|
134
|
+
predict_batch = cast(Callable[[Array], Any], self.predict_batch)
|
|
135
|
+
start = 0
|
|
136
|
+
while start < len(job.ids):
|
|
137
|
+
ids = job.ids[start : start + self.cap]
|
|
138
|
+
try:
|
|
139
|
+
raw = predict_batch(job.rows[start : start + len(ids)].copy())
|
|
140
|
+
except Exception as exc:
|
|
141
|
+
torch = sys.modules.get("torch")
|
|
142
|
+
if (
|
|
143
|
+
self.adaptive
|
|
144
|
+
and len(ids) > 1
|
|
145
|
+
and torch is not None
|
|
146
|
+
and isinstance(exc, torch.cuda.OutOfMemoryError)
|
|
147
|
+
):
|
|
148
|
+
self.cap = max(1, len(ids) // 2)
|
|
149
|
+
continue
|
|
150
|
+
reply.errors.update(dict.fromkeys(ids, describe(exc)))
|
|
151
|
+
break
|
|
152
|
+
try:
|
|
153
|
+
self._score(ids, raw, reply)
|
|
154
|
+
except Exception as exc:
|
|
155
|
+
reply.errors.update(dict.fromkeys(ids, describe(exc)))
|
|
156
|
+
del raw # Release device output before allocating the next batch.
|
|
157
|
+
if reply.errors:
|
|
158
|
+
break
|
|
159
|
+
start += len(ids)
|
|
160
|
+
|
|
161
|
+
def __call__(self, job: Job) -> Reply:
|
|
162
|
+
reply = Reply(cap=self.cap)
|
|
163
|
+
if self.objective.preprocess is not None:
|
|
164
|
+
prepared = prepare(job, self.objective.preprocess)
|
|
165
|
+
reply.errors.update(prepared.errors)
|
|
166
|
+
reply.widths.update(prepared.widths)
|
|
167
|
+
if not prepared.prepared:
|
|
168
|
+
return reply
|
|
169
|
+
width = next(iter(prepared.widths.values()))
|
|
170
|
+
ids = []
|
|
171
|
+
rows = []
|
|
172
|
+
for index, row in prepared.prepared.items():
|
|
173
|
+
if len(row) != width:
|
|
174
|
+
reply.errors[index] = "preprocess must return fixed-length vectors"
|
|
175
|
+
else:
|
|
176
|
+
ids.append(index)
|
|
177
|
+
rows.append(row)
|
|
178
|
+
job = Job(tuple(ids), np.stack(rows))
|
|
179
|
+
self._inference(job, reply)
|
|
180
|
+
reply.cap = self.cap
|
|
181
|
+
return reply
|
labga/_resources.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Process-aware CPU budget detection, including common Linux cgroup layouts."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _quota(directory: Path) -> int | None:
|
|
9
|
+
try:
|
|
10
|
+
if (directory / "cpu.max").is_file():
|
|
11
|
+
quota_text, period_text = (directory / "cpu.max").read_text().split()
|
|
12
|
+
else:
|
|
13
|
+
quota_text = (directory / "cpu.cfs_quota_us").read_text()
|
|
14
|
+
period_text = (directory / "cpu.cfs_period_us").read_text()
|
|
15
|
+
quota, period = int(quota_text), int(period_text)
|
|
16
|
+
if quota > 0 and period > 0:
|
|
17
|
+
return max(1, math.ceil(quota / period))
|
|
18
|
+
except (OSError, ValueError):
|
|
19
|
+
pass
|
|
20
|
+
return None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _cgroup_limit(root: Path = Path("/sys/fs/cgroup")) -> int | None:
|
|
24
|
+
directories = {root, root / "cpu", root / "cpu,cpuacct", root / "cpuacct,cpu"}
|
|
25
|
+
try:
|
|
26
|
+
for line in Path("/proc/self/cgroup").read_text().splitlines():
|
|
27
|
+
_, controllers, path = line.split(":", 2)
|
|
28
|
+
if controllers and "cpu" not in controllers.split(","):
|
|
29
|
+
continue
|
|
30
|
+
base = root / controllers if controllers else root
|
|
31
|
+
relative = Path(path.lstrip("/"))
|
|
32
|
+
if ".." in relative.parts:
|
|
33
|
+
continue
|
|
34
|
+
directories.update(base / part for part in (relative, *relative.parents))
|
|
35
|
+
except (OSError, ValueError):
|
|
36
|
+
pass
|
|
37
|
+
quotas = [value for directory in directories if (value := _quota(directory)) is not None]
|
|
38
|
+
return min(quotas, default=None)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def available_cpus() -> tuple[int, dict[str, int | None]]:
|
|
42
|
+
"""Return the effective budget and the individual observations behind it."""
|
|
43
|
+
process_count = getattr(os, "process_cpu_count", None)
|
|
44
|
+
process = process_count() if process_count else None
|
|
45
|
+
affinity_fn = getattr(os, "sched_getaffinity", None)
|
|
46
|
+
try:
|
|
47
|
+
affinity = len(affinity_fn(0)) if affinity_fn else None
|
|
48
|
+
except OSError:
|
|
49
|
+
affinity = None
|
|
50
|
+
system = os.cpu_count()
|
|
51
|
+
quota = _cgroup_limit()
|
|
52
|
+
available = process or affinity or system or 1
|
|
53
|
+
if affinity is not None:
|
|
54
|
+
available = min(available, affinity)
|
|
55
|
+
if quota is not None:
|
|
56
|
+
available = min(available, quota)
|
|
57
|
+
return max(1, available), {
|
|
58
|
+
"process": process,
|
|
59
|
+
"affinity": affinity,
|
|
60
|
+
"system": system,
|
|
61
|
+
"quota": quota,
|
|
62
|
+
}
|
labga/_types.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Internal array and search-space contracts."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Protocol
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
from numpy.random import Generator
|
|
7
|
+
from numpy.typing import ArrayLike, NDArray
|
|
8
|
+
|
|
9
|
+
FloatArray = NDArray[np.float64]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Genome(Protocol):
|
|
13
|
+
"""Minimal search-space behavior consumed by the optimizer."""
|
|
14
|
+
|
|
15
|
+
@property
|
|
16
|
+
def size(self) -> int: ...
|
|
17
|
+
|
|
18
|
+
def sample(self, count: int, rng: Generator) -> FloatArray: ...
|
|
19
|
+
|
|
20
|
+
def validate(self, population: ArrayLike) -> FloatArray: ...
|
|
21
|
+
|
|
22
|
+
def mutate(
|
|
23
|
+
self, population: FloatArray, rng: Generator, probability: float, scale: float
|
|
24
|
+
) -> FloatArray: ...
|
|
25
|
+
|
|
26
|
+
def encode(self, candidate: FloatArray) -> NDArray[Any]: ...
|
labga/_worker_pool.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""A spawn pool with one outstanding request per child and bounded waits."""
|
|
2
|
+
|
|
3
|
+
import multiprocessing as mp
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
7
|
+
from concurrent.futures import FIRST_COMPLETED, Future, wait
|
|
8
|
+
from multiprocessing.connection import Connection
|
|
9
|
+
from multiprocessing.process import BaseProcess
|
|
10
|
+
from queue import SimpleQueue
|
|
11
|
+
from threading import Thread
|
|
12
|
+
|
|
13
|
+
from threadpoolctl import threadpool_limits
|
|
14
|
+
|
|
15
|
+
from labga._evaluation_worker import Evaluator, Job, Reply, WorkerSpec, describe, prepare
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _serve(wire: Connection, spec: WorkerSpec) -> None:
|
|
19
|
+
try:
|
|
20
|
+
with threadpool_limits(limits=1):
|
|
21
|
+
run: Callable[[Job], Reply]
|
|
22
|
+
if spec.objective is None:
|
|
23
|
+
assert spec.preprocess is not None
|
|
24
|
+
transform = spec.preprocess
|
|
25
|
+
|
|
26
|
+
def run(job: Job) -> Reply:
|
|
27
|
+
return prepare(job, transform)
|
|
28
|
+
|
|
29
|
+
batched = True
|
|
30
|
+
else:
|
|
31
|
+
owner = Evaluator(spec.objective, spec.cap)
|
|
32
|
+
run = owner
|
|
33
|
+
batched = owner.batched
|
|
34
|
+
torch = sys.modules.get("torch")
|
|
35
|
+
if torch is not None:
|
|
36
|
+
torch.set_num_threads(1)
|
|
37
|
+
# Apply the limit again to native libraries imported by the factory.
|
|
38
|
+
with threadpool_limits(limits=1):
|
|
39
|
+
wire.send(batched)
|
|
40
|
+
while (job := wire.recv()) is not None:
|
|
41
|
+
wire.send(run(job))
|
|
42
|
+
except (EOFError, BrokenPipeError):
|
|
43
|
+
pass
|
|
44
|
+
except BaseException as exc:
|
|
45
|
+
try:
|
|
46
|
+
wire.send(describe(exc))
|
|
47
|
+
except (OSError, EOFError):
|
|
48
|
+
pass
|
|
49
|
+
finally:
|
|
50
|
+
wire.close()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# The timestamp is captured after reading and unpickling the entire message.
|
|
54
|
+
_Response = Future[tuple[float, object]]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class Child:
|
|
58
|
+
"""Exchange messages on one dedicated thread so partial frames never block the scheduler."""
|
|
59
|
+
|
|
60
|
+
def __init__(self, process: BaseProcess, wire: Connection) -> None:
|
|
61
|
+
self.process = process
|
|
62
|
+
self.wire = wire
|
|
63
|
+
self.ready: _Response = Future()
|
|
64
|
+
self.requests: SimpleQueue[tuple[Job, _Response] | None] = SimpleQueue()
|
|
65
|
+
self.thread = Thread(target=self._exchange, name=f"labga-ipc-{process.pid}", daemon=True)
|
|
66
|
+
|
|
67
|
+
def _exchange(self) -> None:
|
|
68
|
+
response = self.ready
|
|
69
|
+
try:
|
|
70
|
+
status = self.wire.recv()
|
|
71
|
+
response.set_result((time.monotonic(), status))
|
|
72
|
+
while (request := self.requests.get()) is not None:
|
|
73
|
+
job, response = request
|
|
74
|
+
self.wire.send(job)
|
|
75
|
+
reply = self.wire.recv()
|
|
76
|
+
response.set_result((time.monotonic(), reply))
|
|
77
|
+
except Exception as exc:
|
|
78
|
+
response.set_result((time.monotonic(), exc))
|
|
79
|
+
finally:
|
|
80
|
+
# Closing the last parent endpoint also tells an idle child to exit.
|
|
81
|
+
self.wire.close()
|
|
82
|
+
|
|
83
|
+
def submit(self, job: Job) -> _Response:
|
|
84
|
+
response: _Response = Future()
|
|
85
|
+
self.requests.put((job, response))
|
|
86
|
+
return response
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _responses(deadlines: dict[_Response, float]) -> Iterator[tuple[_Response, object]]:
|
|
90
|
+
"""Yield completed or timed-out exchanges, including ones completed after their deadline."""
|
|
91
|
+
delay = max(0.0, min(deadlines.values()) - time.monotonic())
|
|
92
|
+
wait(deadlines, delay, FIRST_COMPLETED)
|
|
93
|
+
now = time.monotonic()
|
|
94
|
+
for future, deadline in list(deadlines.items()):
|
|
95
|
+
result: object = TimeoutError("timed out")
|
|
96
|
+
if future.done():
|
|
97
|
+
received, payload = future.result()
|
|
98
|
+
if received <= deadline:
|
|
99
|
+
result = payload
|
|
100
|
+
elif deadline > now:
|
|
101
|
+
continue
|
|
102
|
+
del deadlines[future]
|
|
103
|
+
yield future, result
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class SpawnPool:
|
|
107
|
+
"""Own every process and its transport, including partially initialized children."""
|
|
108
|
+
|
|
109
|
+
def __init__(self, spec: WorkerSpec, timeout: float) -> None:
|
|
110
|
+
self.spec = spec
|
|
111
|
+
self.timeout = timeout
|
|
112
|
+
self.children: list[Child] = []
|
|
113
|
+
self.batched: bool | None = None
|
|
114
|
+
|
|
115
|
+
def ensure(self, count: int) -> None:
|
|
116
|
+
ctx = mp.get_context("spawn")
|
|
117
|
+
starting: dict[_Response, float] = {}
|
|
118
|
+
try:
|
|
119
|
+
while len(self.children) < count:
|
|
120
|
+
parent, child_wire = ctx.Pipe()
|
|
121
|
+
process = ctx.Process(target=_serve, args=(child_wire, self.spec))
|
|
122
|
+
deadline = time.monotonic() + self.timeout
|
|
123
|
+
try:
|
|
124
|
+
process.start()
|
|
125
|
+
except BaseException:
|
|
126
|
+
parent.close()
|
|
127
|
+
raise
|
|
128
|
+
finally:
|
|
129
|
+
child_wire.close()
|
|
130
|
+
child = Child(process, parent)
|
|
131
|
+
self.children.append(child)
|
|
132
|
+
starting[child.ready] = deadline
|
|
133
|
+
child.thread.start()
|
|
134
|
+
|
|
135
|
+
while starting:
|
|
136
|
+
for _, status in _responses(starting):
|
|
137
|
+
if isinstance(status, TimeoutError):
|
|
138
|
+
raise RuntimeError("worker initialization timed out")
|
|
139
|
+
if isinstance(status, EOFError):
|
|
140
|
+
raise RuntimeError("worker exited during initialization") from status
|
|
141
|
+
if not isinstance(status, bool):
|
|
142
|
+
raise RuntimeError(f"worker initialization failed: {status}")
|
|
143
|
+
if self.batched is not None and self.batched != status:
|
|
144
|
+
raise RuntimeError("factory returned inconsistent batch capabilities")
|
|
145
|
+
self.batched = status
|
|
146
|
+
except BaseException:
|
|
147
|
+
self.close(abort=True)
|
|
148
|
+
raise
|
|
149
|
+
|
|
150
|
+
def run(self, jobs: Iterable[Job]) -> Iterator[Reply]:
|
|
151
|
+
remaining = iter(jobs)
|
|
152
|
+
idle = list(reversed(self.children))
|
|
153
|
+
pending: dict[_Response, tuple[Child, Job]] = {}
|
|
154
|
+
deadlines: dict[_Response, float] = {}
|
|
155
|
+
submitting = True
|
|
156
|
+
while pending or (idle and submitting):
|
|
157
|
+
while idle and submitting:
|
|
158
|
+
job = next(remaining, None)
|
|
159
|
+
if job is None:
|
|
160
|
+
submitting = False
|
|
161
|
+
break
|
|
162
|
+
child = idle.pop()
|
|
163
|
+
deadline = time.monotonic() + self.timeout
|
|
164
|
+
future = child.submit(job)
|
|
165
|
+
pending[future] = (child, job)
|
|
166
|
+
deadlines[future] = deadline
|
|
167
|
+
if not pending:
|
|
168
|
+
break
|
|
169
|
+
for future, result in _responses(deadlines):
|
|
170
|
+
child, job = pending.pop(future)
|
|
171
|
+
if not isinstance(result, Reply):
|
|
172
|
+
message = (
|
|
173
|
+
"worker request timed out"
|
|
174
|
+
if isinstance(result, TimeoutError)
|
|
175
|
+
else f"worker exited: {result}"
|
|
176
|
+
)
|
|
177
|
+
result = Reply(errors=dict.fromkeys(job.ids, message))
|
|
178
|
+
if result.errors:
|
|
179
|
+
submitting = False
|
|
180
|
+
else:
|
|
181
|
+
idle.append(child)
|
|
182
|
+
yield result
|
|
183
|
+
|
|
184
|
+
def close(self, *, abort: bool = False) -> None:
|
|
185
|
+
for child in self.children:
|
|
186
|
+
child.requests.put(None)
|
|
187
|
+
if abort and child.process.is_alive():
|
|
188
|
+
child.process.terminate()
|
|
189
|
+
end = time.monotonic() + 2
|
|
190
|
+
for child in self.children:
|
|
191
|
+
child.process.join(max(0.0, end - time.monotonic()))
|
|
192
|
+
for child in self.children:
|
|
193
|
+
if child.process.is_alive():
|
|
194
|
+
child.process.kill()
|
|
195
|
+
end = time.monotonic() + 2
|
|
196
|
+
for child in self.children:
|
|
197
|
+
child.process.join(max(0.0, end - time.monotonic()))
|
|
198
|
+
if not child.process.is_alive():
|
|
199
|
+
child.process.close()
|
|
200
|
+
if child.thread.ident is not None:
|
|
201
|
+
child.thread.join(max(0.0, end - time.monotonic()))
|
|
202
|
+
else:
|
|
203
|
+
child.wire.close()
|
|
204
|
+
self.children.clear()
|