fuzzybits-server 0.17.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fuzzybits_server/.dockerignore +64 -0
- fuzzybits_server/__init__.py +15 -0
- fuzzybits_server/app.py +93 -0
- fuzzybits_server/config.py +310 -0
- fuzzybits_server/convergence/__init__.py +100 -0
- fuzzybits_server/convergence/gbas.py +229 -0
- fuzzybits_server/convergence/metrics.py +164 -0
- fuzzybits_server/convergence/policies.py +387 -0
- fuzzybits_server/convergence/runner.py +398 -0
- fuzzybits_server/convergence/sampling.py +409 -0
- fuzzybits_server/convergence/special.py +280 -0
- fuzzybits_server/convergence/tally.py +179 -0
- fuzzybits_server/event_hub.py +58 -0
- fuzzybits_server/event_log.py +16 -0
- fuzzybits_server/fault_tree_payload.py +899 -0
- fuzzybits_server/model_oracle_p.json +44 -0
- fuzzybits_server/native.py +536 -0
- fuzzybits_server/routes.py +408 -0
- fuzzybits_server/state.py +67 -0
- fuzzybits_server/supervisor.py +193 -0
- fuzzybits_server/ui.py +84 -0
- fuzzybits_server/webui/.vite/manifest.json +26 -0
- fuzzybits_server/webui/assets/Playground-B6V8NzhX.css +1 -0
- fuzzybits_server/webui/assets/Playground-CSpFbzwZ.js +5481 -0
- fuzzybits_server/webui/assets/Playground-CSpFbzwZ.js.map +1 -0
- fuzzybits_server/webui/assets/index-BnegRRhI.css +1 -0
- fuzzybits_server/webui/assets/index-C3oGCBE0.js +13 -0
- fuzzybits_server/webui/assets/index-C3oGCBE0.js.map +1 -0
- fuzzybits_server/webui/index.html +24 -0
- fuzzybits_server/worker.py +64 -0
- fuzzybits_server-0.17.0.dist-info/METADATA +281 -0
- fuzzybits_server-0.17.0.dist-info/RECORD +34 -0
- fuzzybits_server-0.17.0.dist-info/WHEEL +4 -0
- fuzzybits_server-0.17.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"baobab1": 0.000101708,
|
|
3
|
+
"baobab2": 0.000713018,
|
|
4
|
+
"baobab3": 0.00224117,
|
|
5
|
+
"cea9601": 0.00148409,
|
|
6
|
+
"chinese": 0.00117058,
|
|
7
|
+
"das9201": 0.01342366773,
|
|
8
|
+
"das9202": 0.01011538126,
|
|
9
|
+
"das9203": 0.0013488,
|
|
10
|
+
"das9204": 2.16942e-11,
|
|
11
|
+
"das9205": 1.38408e-08,
|
|
12
|
+
"das9206": 0.229686838,
|
|
13
|
+
"das9207": 0.3466958884,
|
|
14
|
+
"das9208": 0.01301789692,
|
|
15
|
+
"das9209": 1.058e-13,
|
|
16
|
+
"das9601": 0.0042344,
|
|
17
|
+
"das9701": 0.0744694,
|
|
18
|
+
"edf9201": 0.3245914467,
|
|
19
|
+
"edf9202": 0.7813024513,
|
|
20
|
+
"edf9203": 0.5995890977,
|
|
21
|
+
"edf9204": 0.5253742885,
|
|
22
|
+
"edf9205": 0.2093509058,
|
|
23
|
+
"edf9206": 8.615e-12,
|
|
24
|
+
"edfpa14b": 0.2956195457,
|
|
25
|
+
"edfpa14o": 0.2970571108,
|
|
26
|
+
"edfpa14p": 0.08070592177,
|
|
27
|
+
"edfpa14q": 0.2959054909,
|
|
28
|
+
"edfpa14r": 0.02099765778,
|
|
29
|
+
"edfpa15b": 0.3627365169,
|
|
30
|
+
"edfpa15o": 0.3629559152,
|
|
31
|
+
"edfpa15p": 0.07363023823,
|
|
32
|
+
"edfpa15q": 0.3627365169,
|
|
33
|
+
"edfpa15r": 0.01897503071,
|
|
34
|
+
"elf9601": 0.09662909854,
|
|
35
|
+
"ftr10": 0.4486771197,
|
|
36
|
+
"isp9601": 0.05712449272,
|
|
37
|
+
"isp9602": 0.01724474483,
|
|
38
|
+
"isp9603": 0.003233264387,
|
|
39
|
+
"isp9604": 0.1427507476,
|
|
40
|
+
"isp9605": 1.37171e-05,
|
|
41
|
+
"isp9606": 0.05431735536,
|
|
42
|
+
"isp9607": 9.4951e-07,
|
|
43
|
+
"jbd9601": 0.7550906151
|
|
44
|
+
}
|
|
@@ -0,0 +1,536 @@
|
|
|
1
|
+
"""Isolated wrappers around the fuzzybits native extension.
|
|
2
|
+
|
|
3
|
+
AdaptiveCpp and CUDA failures can abort the interpreter via ``std::terminate``,
|
|
4
|
+
which Python exception handlers cannot catch. Compilation therefore runs in a
|
|
5
|
+
single long-lived child worker so the API server survives device errors and
|
|
6
|
+
concurrent requests do not stampede the GPU allocator.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import importlib.util
|
|
13
|
+
import json
|
|
14
|
+
import multiprocessing as mp
|
|
15
|
+
import os
|
|
16
|
+
import queue
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
import traceback
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from enum import Enum
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Callable
|
|
24
|
+
|
|
25
|
+
from fuzzybits_server.config import parse_watch_value, pcfb_cache_dir, server_engine
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CompileProduct(str, Enum):
|
|
29
|
+
GRAPH = "graph"
|
|
30
|
+
KERNEL_GRAPH = "kernel_graph"
|
|
31
|
+
EXECUTION_PLAN = "execution_plan"
|
|
32
|
+
SAMPLE_SHAPE = "sample_shape"
|
|
33
|
+
BUNDLE = "bundle"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class NativeResult:
|
|
38
|
+
ok: bool
|
|
39
|
+
payload: Any = None
|
|
40
|
+
error: str | None = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class InvalidPcfbError(ValueError):
|
|
44
|
+
"""Uploaded / loaded `.pcfb` failed `fuzzybits.inspect` / `pcfb::validate`."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, message: str, diagnostics: list[Any] | None = None, inspect: dict[str, Any] | None = None):
|
|
47
|
+
super().__init__(message)
|
|
48
|
+
self.diagnostics = diagnostics or []
|
|
49
|
+
self.inspect = inspect or {}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
_COMPILE_CACHE_MAX = 8
|
|
53
|
+
|
|
54
|
+
# Host-side keys that run_compile's settings_from_options must not see.
|
|
55
|
+
# `watch` as a name list is not watch_mode; booleans stay as watch_mode.
|
|
56
|
+
_HOST_ONLY_OPTION_KEYS: frozenset[str] = frozenset(
|
|
57
|
+
{
|
|
58
|
+
"policy",
|
|
59
|
+
"P",
|
|
60
|
+
"early_stop",
|
|
61
|
+
"max_passes",
|
|
62
|
+
"emit_min_interval_s",
|
|
63
|
+
"stream_nodes",
|
|
64
|
+
"convergence_event_ids",
|
|
65
|
+
"pcfb_path",
|
|
66
|
+
"pcfb_id",
|
|
67
|
+
}
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
# Options that change the lowered circuit (cache key + compile_pcfb).
|
|
71
|
+
_COMPILE_OPTION_KEYS: frozenset[str] = frozenset(
|
|
72
|
+
{
|
|
73
|
+
"compilation_level",
|
|
74
|
+
"compilation_passes",
|
|
75
|
+
"pdag_c",
|
|
76
|
+
"c",
|
|
77
|
+
"no_kn",
|
|
78
|
+
"expand_atleast_gates",
|
|
79
|
+
"expand_kn",
|
|
80
|
+
"no_xor",
|
|
81
|
+
"expand_xor_gates",
|
|
82
|
+
"recognize_atleast_groups",
|
|
83
|
+
"recognize_xor_groups",
|
|
84
|
+
"keep_null_gates",
|
|
85
|
+
"ccf",
|
|
86
|
+
"allow_extern",
|
|
87
|
+
"target",
|
|
88
|
+
"target_gate",
|
|
89
|
+
"top",
|
|
90
|
+
"top_event",
|
|
91
|
+
"watch",
|
|
92
|
+
"watch_nodes",
|
|
93
|
+
}
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def format_native_exception(exc: BaseException) -> str:
|
|
98
|
+
return "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def import_native() -> Any:
|
|
102
|
+
import fuzzybits
|
|
103
|
+
|
|
104
|
+
return fuzzybits.native()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def native_import_error() -> str | None:
|
|
108
|
+
if importlib.util.find_spec("fuzzybits") is None:
|
|
109
|
+
return "fuzzybits is not installed"
|
|
110
|
+
try:
|
|
111
|
+
import fuzzybits
|
|
112
|
+
|
|
113
|
+
fuzzybits.native()
|
|
114
|
+
except Exception as exc:
|
|
115
|
+
return f"fuzzybits native extension is not available: {exc}"
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def run_probability_call(
|
|
120
|
+
input_files: list[str],
|
|
121
|
+
options: dict[str, Any],
|
|
122
|
+
on_progress: Callable[[dict[str, Any]], None],
|
|
123
|
+
) -> dict[str, Any]:
|
|
124
|
+
"""Run Monte Carlo probability analysis in the current process.
|
|
125
|
+
|
|
126
|
+
Intended for the dedicated calculation subprocess only. Device failures may
|
|
127
|
+
still abort this process; the API server stays isolated via
|
|
128
|
+
:mod:`fuzzybits_server.worker`.
|
|
129
|
+
"""
|
|
130
|
+
native = import_native()
|
|
131
|
+
return native.run_probability(input_files, options, on_progress)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _native_timeout_seconds() -> float:
|
|
135
|
+
raw = os.getenv("MCSCRAM_NATIVE_TIMEOUT_SEC", "600")
|
|
136
|
+
try:
|
|
137
|
+
return float(raw)
|
|
138
|
+
except ValueError:
|
|
139
|
+
return 600.0
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _compile_cache_key(input_files: list[str], options: dict[str, Any]) -> tuple[Any, ...]:
|
|
143
|
+
return (tuple(input_files), tuple(sorted(options.items())))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def compile_option_subset(options: dict[str, Any] | None) -> dict[str, Any]:
|
|
147
|
+
if not options:
|
|
148
|
+
return {}
|
|
149
|
+
out: dict[str, Any] = {}
|
|
150
|
+
for key, value in options.items():
|
|
151
|
+
if key in _COMPILE_OPTION_KEYS:
|
|
152
|
+
out[key] = value
|
|
153
|
+
return out
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def watch_names_from_options(options: dict[str, Any] | None) -> list[str]:
|
|
157
|
+
if not options:
|
|
158
|
+
return []
|
|
159
|
+
_, names = parse_watch_value(options.get("watch"))
|
|
160
|
+
extra = options.get("watch_nodes")
|
|
161
|
+
if extra:
|
|
162
|
+
_, more = parse_watch_value(extra)
|
|
163
|
+
names = names + more
|
|
164
|
+
return names
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def pcfb_cache_digest(input_files: list[str], options: dict[str, Any] | None) -> str:
|
|
168
|
+
hasher = hashlib.sha256()
|
|
169
|
+
for path in input_files:
|
|
170
|
+
payload = Path(path).read_bytes()
|
|
171
|
+
hasher.update(path.encode())
|
|
172
|
+
hasher.update(b"\0")
|
|
173
|
+
hasher.update(payload)
|
|
174
|
+
hasher.update(b"\0")
|
|
175
|
+
canon = json.dumps(compile_option_subset(options), sort_keys=True, default=str)
|
|
176
|
+
hasher.update(canon.encode())
|
|
177
|
+
return hasher.hexdigest()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def cached_pcfb_path(digest: str) -> Path:
|
|
181
|
+
return pcfb_cache_dir() / f"{digest}.pcfb"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def compile_pcfb_to_cache(input_files: list[str], options: dict[str, Any] | None) -> bytes:
|
|
185
|
+
"""Compile MEF to `.pcfb`, keyed by source SHA-256s + compile options."""
|
|
186
|
+
if not input_files:
|
|
187
|
+
raise ValueError("compile requires at least one input file")
|
|
188
|
+
digest = pcfb_cache_digest(input_files, options)
|
|
189
|
+
path = cached_pcfb_path(digest)
|
|
190
|
+
if path.is_file():
|
|
191
|
+
return path.read_bytes()
|
|
192
|
+
native = import_native()
|
|
193
|
+
blob = bytes(native.compile_pcfb(input_files, compile_option_subset(options) or {}))
|
|
194
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
195
|
+
tmp = path.with_suffix(".pcfb.tmp")
|
|
196
|
+
tmp.write_bytes(blob)
|
|
197
|
+
tmp.replace(path)
|
|
198
|
+
return blob
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def inspect_pcfb_bytes(data: bytes) -> dict[str, Any]:
|
|
202
|
+
try:
|
|
203
|
+
import fuzzybits
|
|
204
|
+
|
|
205
|
+
return dict(fuzzybits.inspect(data))
|
|
206
|
+
except ImportError:
|
|
207
|
+
native = import_native()
|
|
208
|
+
return dict(native.inspect_pcfb_bytes(data))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def uploaded_pcfb_dir() -> Path:
|
|
212
|
+
path = pcfb_cache_dir() / "uploads"
|
|
213
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
214
|
+
return path
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def resolve_uploaded_pcfb(pcfb_id: str) -> Path:
|
|
218
|
+
digest = pcfb_id.strip().lower().removeprefix("pcfb:")
|
|
219
|
+
if not digest or any(c not in "0123456789abcdef" for c in digest):
|
|
220
|
+
raise ValueError(f"invalid pcfb id: {pcfb_id}")
|
|
221
|
+
path = uploaded_pcfb_dir() / f"{digest}.pcfb"
|
|
222
|
+
if not path.is_file():
|
|
223
|
+
path = cached_pcfb_path(digest)
|
|
224
|
+
if not path.is_file():
|
|
225
|
+
raise FileNotFoundError(f"no uploaded .pcfb with id {digest}")
|
|
226
|
+
return path
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def store_uploaded_pcfb(data: bytes) -> tuple[str, Path, dict[str, Any]]:
|
|
230
|
+
if not data:
|
|
231
|
+
raise InvalidPcfbError("empty pcfb upload")
|
|
232
|
+
try:
|
|
233
|
+
summary = inspect_pcfb_bytes(data)
|
|
234
|
+
except BaseException as exc:
|
|
235
|
+
raise InvalidPcfbError(f"invalid pcfb: {exc}") from exc
|
|
236
|
+
diagnostics = list(summary.get("diagnostics") or [])
|
|
237
|
+
if diagnostics:
|
|
238
|
+
raise InvalidPcfbError("pcfb validation failed", diagnostics=diagnostics, inspect=summary)
|
|
239
|
+
digest = hashlib.sha256(data).hexdigest()
|
|
240
|
+
path = uploaded_pcfb_dir() / f"{digest}.pcfb"
|
|
241
|
+
if not path.is_file():
|
|
242
|
+
tmp = path.with_suffix(".pcfb.tmp")
|
|
243
|
+
tmp.write_bytes(data)
|
|
244
|
+
tmp.replace(path)
|
|
245
|
+
return digest, path, summary
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _trials_per_pass(options: dict[str, Any] | None) -> int | None:
|
|
249
|
+
if not options:
|
|
250
|
+
return None
|
|
251
|
+
raw = options.get("requested_trials_per_pass", options.get("trials_per_pass"))
|
|
252
|
+
if raw is None:
|
|
253
|
+
return None
|
|
254
|
+
value = int(float(raw))
|
|
255
|
+
return value if value > 0 else None
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _num_trials(options: dict[str, Any] | None) -> int | None:
|
|
259
|
+
if not options:
|
|
260
|
+
return None
|
|
261
|
+
raw = options.get("num_trials")
|
|
262
|
+
if raw is None:
|
|
263
|
+
return None
|
|
264
|
+
value = int(float(raw))
|
|
265
|
+
return value if value > 0 else None
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _session_from_pcfb(native: Any, blob: bytes, options: dict[str, Any] | None) -> Any:
|
|
269
|
+
return native.PcfbSession(
|
|
270
|
+
blob,
|
|
271
|
+
"",
|
|
272
|
+
_trials_per_pass(options),
|
|
273
|
+
watch_names_from_options(options),
|
|
274
|
+
_num_trials(options),
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _payload_from_pcfb_session(session: Any, product: str) -> Any:
|
|
279
|
+
match product:
|
|
280
|
+
case CompileProduct.GRAPH.value:
|
|
281
|
+
return dict(session.graph())
|
|
282
|
+
case CompileProduct.KERNEL_GRAPH.value:
|
|
283
|
+
return dict(session.kernel_graph())
|
|
284
|
+
case CompileProduct.EXECUTION_PLAN.value:
|
|
285
|
+
return dict(session.execution_plan())
|
|
286
|
+
case CompileProduct.SAMPLE_SHAPE.value:
|
|
287
|
+
return {"sample_shape": dict(session.sample_shape)}
|
|
288
|
+
case CompileProduct.BUNDLE.value:
|
|
289
|
+
return {
|
|
290
|
+
"compiled": dict(session.graph()),
|
|
291
|
+
"kernel": dict(session.kernel_graph()),
|
|
292
|
+
}
|
|
293
|
+
case "pcfb":
|
|
294
|
+
return None
|
|
295
|
+
case _:
|
|
296
|
+
raise ValueError(f"unknown compile product: {product}")
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _silence_native_stderr() -> None:
|
|
300
|
+
try:
|
|
301
|
+
devnull = os.open(os.devnull, os.O_WRONLY)
|
|
302
|
+
os.dup2(devnull, 2)
|
|
303
|
+
os.close(devnull)
|
|
304
|
+
except OSError:
|
|
305
|
+
pass
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _payload_from_pdag(pdag: Any, product: str) -> Any:
|
|
309
|
+
match product:
|
|
310
|
+
case CompileProduct.GRAPH.value:
|
|
311
|
+
return pdag.to_graph()
|
|
312
|
+
case CompileProduct.KERNEL_GRAPH.value:
|
|
313
|
+
return pdag.to_kernel_graph()
|
|
314
|
+
case CompileProduct.EXECUTION_PLAN.value:
|
|
315
|
+
return dict(pdag.to_execution_plan())
|
|
316
|
+
case CompileProduct.SAMPLE_SHAPE.value:
|
|
317
|
+
layer_manager = pdag.create_layer_manager()
|
|
318
|
+
return {"sample_shape": dict(layer_manager.sample_shape)}
|
|
319
|
+
case CompileProduct.BUNDLE.value:
|
|
320
|
+
return {
|
|
321
|
+
"compiled": pdag.to_graph(),
|
|
322
|
+
"kernel": pdag.to_kernel_graph(),
|
|
323
|
+
}
|
|
324
|
+
case _:
|
|
325
|
+
raise ValueError(f"unknown compile product: {product}")
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _persistent_compile_worker(request_queue: mp.Queue, response_queue: mp.Queue) -> None:
|
|
329
|
+
_silence_native_stderr()
|
|
330
|
+
compile_cache: dict[tuple[Any, ...], Any] = {}
|
|
331
|
+
cache_order: list[tuple[Any, ...]] = []
|
|
332
|
+
pcfb_cache: dict[str, bytes] = {}
|
|
333
|
+
|
|
334
|
+
while True:
|
|
335
|
+
request = request_queue.get()
|
|
336
|
+
if request is None:
|
|
337
|
+
return
|
|
338
|
+
|
|
339
|
+
request_id: int = request["request_id"]
|
|
340
|
+
input_files: list[str] = request["input_files"]
|
|
341
|
+
options: dict[str, Any] = request["options"]
|
|
342
|
+
product: str = request["product"]
|
|
343
|
+
engine: str = request.get("engine") or server_engine()
|
|
344
|
+
pcfb_path: str | None = request.get("pcfb_path")
|
|
345
|
+
cache_key = _compile_cache_key(input_files, options)
|
|
346
|
+
|
|
347
|
+
try:
|
|
348
|
+
if engine == "legacy" and not pcfb_path:
|
|
349
|
+
if cache_key not in compile_cache:
|
|
350
|
+
native = import_native()
|
|
351
|
+
compile_opts = {
|
|
352
|
+
key: value
|
|
353
|
+
for key, value in options.items()
|
|
354
|
+
if key not in _HOST_ONLY_OPTION_KEYS and key not in {"watch", "watch_nodes"}
|
|
355
|
+
}
|
|
356
|
+
mode, _ = parse_watch_value(options.get("watch_mode", options.get("watch")))
|
|
357
|
+
if mode is not None:
|
|
358
|
+
compile_opts["watch_mode"] = mode
|
|
359
|
+
compile_cache[cache_key] = native.run_compile(input_files, compile_opts)
|
|
360
|
+
cache_order.append(cache_key)
|
|
361
|
+
while len(cache_order) > _COMPILE_CACHE_MAX:
|
|
362
|
+
evicted = cache_order.pop(0)
|
|
363
|
+
compile_cache.pop(evicted, None)
|
|
364
|
+
payload = _payload_from_pdag(compile_cache[cache_key], product)
|
|
365
|
+
else:
|
|
366
|
+
native = import_native()
|
|
367
|
+
if pcfb_path:
|
|
368
|
+
blob = Path(pcfb_path).read_bytes()
|
|
369
|
+
else:
|
|
370
|
+
digest = pcfb_cache_digest(input_files, options)
|
|
371
|
+
blob = pcfb_cache.get(digest)
|
|
372
|
+
if blob is None:
|
|
373
|
+
blob = compile_pcfb_to_cache(input_files, options)
|
|
374
|
+
pcfb_cache[digest] = blob
|
|
375
|
+
while len(pcfb_cache) > _COMPILE_CACHE_MAX:
|
|
376
|
+
pcfb_cache.pop(next(iter(pcfb_cache)))
|
|
377
|
+
session = _session_from_pcfb(native, blob, options)
|
|
378
|
+
try:
|
|
379
|
+
payload = _payload_from_pcfb_session(session, product)
|
|
380
|
+
if product == "pcfb":
|
|
381
|
+
payload = blob
|
|
382
|
+
finally:
|
|
383
|
+
close = getattr(session, "close", None)
|
|
384
|
+
if close is not None:
|
|
385
|
+
close()
|
|
386
|
+
response_queue.put({"request_id": request_id, "ok": True, "payload": payload})
|
|
387
|
+
except BaseException as exc:
|
|
388
|
+
compile_cache.pop(cache_key, None)
|
|
389
|
+
if cache_key in cache_order:
|
|
390
|
+
cache_order.remove(cache_key)
|
|
391
|
+
response_queue.put({
|
|
392
|
+
"request_id": request_id,
|
|
393
|
+
"ok": False,
|
|
394
|
+
"error": format_native_exception(exc),
|
|
395
|
+
})
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _result_from_queue_message(message: dict[str, Any] | None) -> NativeResult:
|
|
399
|
+
if message is None:
|
|
400
|
+
return NativeResult(ok=False, error="native compile worker produced no result")
|
|
401
|
+
if not message.get("ok", False):
|
|
402
|
+
return NativeResult(ok=False, error=str(message.get("error", "native compile failed")))
|
|
403
|
+
return NativeResult(ok=True, payload=message.get("payload"))
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
class NativeCompileBroker:
|
|
407
|
+
"""Owns one compile worker process and serializes all native compile calls."""
|
|
408
|
+
|
|
409
|
+
def __init__(self) -> None:
|
|
410
|
+
self._ctx = mp.get_context("spawn")
|
|
411
|
+
self._lock = threading.Lock()
|
|
412
|
+
self._request_queue: mp.Queue | None = None
|
|
413
|
+
self._response_queue: mp.Queue | None = None
|
|
414
|
+
self._worker: mp.Process | None = None
|
|
415
|
+
self._next_request_id = 0
|
|
416
|
+
|
|
417
|
+
def stop(self) -> None:
|
|
418
|
+
with self._lock:
|
|
419
|
+
self._shutdown_worker()
|
|
420
|
+
|
|
421
|
+
def run(
|
|
422
|
+
self,
|
|
423
|
+
input_files: list[str],
|
|
424
|
+
options: dict[str, Any] | None,
|
|
425
|
+
product: CompileProduct,
|
|
426
|
+
*,
|
|
427
|
+
timeout: float | None = None,
|
|
428
|
+
engine: str | None = None,
|
|
429
|
+
pcfb_path: str | None = None,
|
|
430
|
+
) -> NativeResult:
|
|
431
|
+
timeout_seconds = _native_timeout_seconds() if timeout is None else timeout
|
|
432
|
+
with self._lock:
|
|
433
|
+
if not self._ensure_worker():
|
|
434
|
+
return NativeResult(ok=False, error="native compile worker failed to start")
|
|
435
|
+
|
|
436
|
+
assert self._request_queue is not None
|
|
437
|
+
assert self._response_queue is not None
|
|
438
|
+
self._next_request_id += 1
|
|
439
|
+
request_id = self._next_request_id
|
|
440
|
+
self._request_queue.put({
|
|
441
|
+
"request_id": request_id,
|
|
442
|
+
"input_files": input_files,
|
|
443
|
+
"options": options or {},
|
|
444
|
+
"product": product.value,
|
|
445
|
+
"engine": engine or server_engine(),
|
|
446
|
+
"pcfb_path": pcfb_path,
|
|
447
|
+
})
|
|
448
|
+
|
|
449
|
+
deadline = time.monotonic() + timeout_seconds
|
|
450
|
+
while time.monotonic() < deadline:
|
|
451
|
+
worker = self._worker
|
|
452
|
+
if worker is None or not worker.is_alive():
|
|
453
|
+
self._shutdown_worker()
|
|
454
|
+
return NativeResult(
|
|
455
|
+
ok=False,
|
|
456
|
+
error="native compile worker crashed (device or native runtime error)",
|
|
457
|
+
)
|
|
458
|
+
try:
|
|
459
|
+
message = self._response_queue.get(timeout=0.2)
|
|
460
|
+
except queue.Empty:
|
|
461
|
+
continue
|
|
462
|
+
if message.get("request_id") != request_id:
|
|
463
|
+
continue
|
|
464
|
+
return _result_from_queue_message(message)
|
|
465
|
+
|
|
466
|
+
self._shutdown_worker()
|
|
467
|
+
return NativeResult(
|
|
468
|
+
ok=False,
|
|
469
|
+
error=f"native compile timed out after {timeout_seconds:g}s",
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
def _ensure_worker(self) -> bool:
|
|
473
|
+
if self._worker is not None and self._worker.is_alive():
|
|
474
|
+
return True
|
|
475
|
+
self._shutdown_worker()
|
|
476
|
+
self._request_queue = self._ctx.Queue()
|
|
477
|
+
self._response_queue = self._ctx.Queue()
|
|
478
|
+
self._worker = self._ctx.Process(
|
|
479
|
+
target=_persistent_compile_worker,
|
|
480
|
+
args=(self._request_queue, self._response_queue),
|
|
481
|
+
name="fuzzybits-native-compile",
|
|
482
|
+
)
|
|
483
|
+
self._worker.start()
|
|
484
|
+
return self._worker.is_alive()
|
|
485
|
+
|
|
486
|
+
def _shutdown_worker(self) -> None:
|
|
487
|
+
worker = self._worker
|
|
488
|
+
request_queue = self._request_queue
|
|
489
|
+
self._worker = None
|
|
490
|
+
self._request_queue = None
|
|
491
|
+
self._response_queue = None
|
|
492
|
+
|
|
493
|
+
if request_queue is not None:
|
|
494
|
+
try:
|
|
495
|
+
request_queue.put(None)
|
|
496
|
+
except (OSError, ValueError):
|
|
497
|
+
pass
|
|
498
|
+
|
|
499
|
+
if worker is None:
|
|
500
|
+
return
|
|
501
|
+
if worker.is_alive():
|
|
502
|
+
worker.join(timeout=2)
|
|
503
|
+
if worker.is_alive():
|
|
504
|
+
worker.terminate()
|
|
505
|
+
worker.join(timeout=2)
|
|
506
|
+
if worker.is_alive():
|
|
507
|
+
worker.kill()
|
|
508
|
+
worker.join(timeout=2)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
_broker = NativeCompileBroker()
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def run_compile_isolated(
|
|
515
|
+
input_files: list[str],
|
|
516
|
+
options: dict[str, Any] | None,
|
|
517
|
+
product: CompileProduct,
|
|
518
|
+
*,
|
|
519
|
+
timeout: float | None = None,
|
|
520
|
+
engine: str | None = None,
|
|
521
|
+
pcfb_path: str | None = None,
|
|
522
|
+
) -> NativeResult:
|
|
523
|
+
"""Compile MEF input in the native worker and return a serializable product."""
|
|
524
|
+
return _broker.run(
|
|
525
|
+
input_files,
|
|
526
|
+
options,
|
|
527
|
+
product,
|
|
528
|
+
timeout=timeout,
|
|
529
|
+
engine=engine,
|
|
530
|
+
pcfb_path=pcfb_path,
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def shutdown_native_workers() -> None:
|
|
535
|
+
"""Stop background native workers during application shutdown."""
|
|
536
|
+
_broker.stop()
|