vramd 0.3.2__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vramd-0.3.2/src/vramd.egg-info → vramd-0.3.4}/PKG-INFO +1 -1
- {vramd-0.3.2 → vramd-0.3.4}/pyproject.toml +1 -1
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/__init__.py +1 -1
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/backend_manager.py +18 -1
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/dispatcher.py +7 -1
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/server.py +7 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/subprocess_pool.py +136 -59
- {vramd-0.3.2 → vramd-0.3.4/src/vramd.egg-info}/PKG-INFO +1 -1
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/SOURCES.txt +1 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_backend_manager.py +51 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_subprocess_pool.py +86 -1
- vramd-0.3.4/tests/test_version.py +21 -0
- {vramd-0.3.2 → vramd-0.3.4}/LICENSE +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/README.md +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/setup.cfg +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/__main__.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/adapter.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/__init__.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/analysis.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/compare.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/emit.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/hw_auto.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/runner.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/sampler.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/serde.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/cli.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/client.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/data/backends.yaml +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/data/test-image.png +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/data/test-mesh.glb +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/errors.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/footprints.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/gpu.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/hooks.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/idle_evictor.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/job_queue.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/learn.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/logging.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/mcp_server.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/process_guard.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/protocol.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/py.typed +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/registry.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/scheduler.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/stats.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/toolchain.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/top.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/vram_planner.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/worker/__init__.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/worker/adapter.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/worker/protocol.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd/worker/serve.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/dependency_links.txt +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/entry_points.txt +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/requires.txt +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/top_level.txt +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_abort_progress.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_backend_manager_hybrid.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_analysis.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_cli.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_emit_compare.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_hw_auto.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_runner.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_sampler.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_serde.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_cli_debug_stats_bench.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_cli_start.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_gpu_ids_load.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_hooks.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_idle_evictor.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_job_queue.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_job_wal.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_learn.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_learn_integration.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_mcp_server.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_peak_profile.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_phase_progress.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_process_guard.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_protocol_env.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_queue_metrics_eta.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_registry.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_registry_layers.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_respawn.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_runtime_spawn.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_scheduler.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_server.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_stats.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_subprocess_integration.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_subprocess_per_backend_e2e.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_toolchain.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_top.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_unload_frees_vram.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_vram_admit.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_vram_planner.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_vramd_coverage_100.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_worker.py +0 -0
- {vramd-0.3.2 → vramd-0.3.4}/tests/test_zero_vram.py +0 -0
|
@@ -692,7 +692,16 @@ class BackendManager:
|
|
|
692
692
|
measured = self._measured_parts_mib(desc, quant_mode=quant_mode)
|
|
693
693
|
if measured is not None:
|
|
694
694
|
weights_measured, activation_measured = measured
|
|
695
|
-
|
|
695
|
+
measured_mem_eff = bool(
|
|
696
|
+
((getattr(desc, "peak_profile", None) or {}).get("load_kwargs") or {}).get("memory_efficient")
|
|
697
|
+
)
|
|
698
|
+
# O desconto 0.65 só vale quando a medição foi feita SEM mem-eff
|
|
699
|
+
# (realidade FP16) e o request agora pede o modo económico. Se a
|
|
700
|
+
# calibração já foi medida COM memory_efficient (caso paint3d 6g:
|
|
701
|
+
# load_kwargs.memory_efficient=true), a activação medida JÁ reflete
|
|
702
|
+
# o modo — descontar de novo subdimensionava o admit em ~1 GiB e
|
|
703
|
+
# admitia jobs que estouravam (OOM-spin do worker).
|
|
704
|
+
if memory_efficient and not group_offload and not measured_mem_eff:
|
|
696
705
|
activation_measured = max(512, int(activation_measured * _MEMORY_EFFICIENT_ACTIVATION_FACTOR))
|
|
697
706
|
return weights_measured, activation_measured
|
|
698
707
|
|
|
@@ -753,6 +762,14 @@ class BackendManager:
|
|
|
753
762
|
footprint_key: str | None = None,
|
|
754
763
|
) -> int:
|
|
755
764
|
"""Pico = pesos(quant) + activação de inferência + safety."""
|
|
765
|
+
with contextlib.suppress(KeyError):
|
|
766
|
+
vram = getattr(self._registry.descriptor(name), "vram", None) or {}
|
|
767
|
+
admit_peak = vram.get("admit_peak_mib")
|
|
768
|
+
if admit_peak:
|
|
769
|
+
# Recomendação do calibrador: pico REAL medido (pesos +
|
|
770
|
+
# activação + safety). Vence a soma analítica — que pode
|
|
771
|
+
# descontar mem-eff duas vezes e admitir jobs condenados.
|
|
772
|
+
return int(admit_peak)
|
|
756
773
|
weights, activation = self.footprint_parts_mib(
|
|
757
774
|
name,
|
|
758
775
|
quant_mode=quant_mode,
|
|
@@ -38,7 +38,13 @@ def _is_worker_dead(result: dict[str, Any]) -> bool:
|
|
|
38
38
|
if result.get("error_code") == P.ERR_WORKER_DEAD:
|
|
39
39
|
return True
|
|
40
40
|
err = str(result.get("error") or "").lower()
|
|
41
|
-
return
|
|
41
|
+
return (
|
|
42
|
+
"não está vivo" in err
|
|
43
|
+
or "nao esta vivo" in err
|
|
44
|
+
or "worker fechou stdout" in err
|
|
45
|
+
or "eof no load" in err
|
|
46
|
+
or "worker wedged" in err
|
|
47
|
+
)
|
|
42
48
|
|
|
43
49
|
|
|
44
50
|
def _vram_retry_worthwhile(result: dict[str, Any], total_mib: int | None) -> bool:
|
|
@@ -420,6 +420,13 @@ class VramdServer:
|
|
|
420
420
|
if "cancel" in err_txt:
|
|
421
421
|
out.setdefault("error_code", P.ERR_CANCELLED)
|
|
422
422
|
out.setdefault("hint", "Job cancelado (queued ou durante generate).")
|
|
423
|
+
elif "oom" in err_txt:
|
|
424
|
+
out.setdefault("error_code", P.ERR_GENERATE_FAILED)
|
|
425
|
+
out.setdefault(
|
|
426
|
+
"hint",
|
|
427
|
+
"OOM/spin no worker — vê o log vramd-worker-<backend>.log; "
|
|
428
|
+
"considera reduzir tamanho/qualidade do pedido.",
|
|
429
|
+
)
|
|
423
430
|
elif "timeout" in err_txt:
|
|
424
431
|
out.setdefault("error_code", P.ERR_TIMEOUT)
|
|
425
432
|
out.setdefault("hint", "Aumenta timeout do cliente ou inspecciona vramd queue.")
|
|
@@ -53,7 +53,18 @@ _logger = Logger()
|
|
|
53
53
|
|
|
54
54
|
# Idle entre eventos de progresso (reset a cada progress). Hunyuan pode
|
|
55
55
|
# demorar muitos minutos no total — NÃO é wall-clock do generate inteiro.
|
|
56
|
-
DEFAULT_EVENT_TIMEOUT_SEC = 600.0
|
|
56
|
+
DEFAULT_EVENT_TIMEOUT_SEC = 600.0
|
|
57
|
+
|
|
58
|
+
# Watcher de OOM-spin: o allocator CUDA em retry infinito não emite eventos —
|
|
59
|
+
# cresce no stderr do worker (~0.7 linhas/s de "expandable_segments: memory
|
|
60
|
+
# mapping failed"). 8 linhas + 30s sem progress denunciam o wedge em ~40s
|
|
61
|
+
# em vez dos 600s do idle-timeout (que depois virava "timeout do cliente").
|
|
62
|
+
_OOM_SPIN_PATTERNS = ("expandable_segments: memory mapping failed", "CUDA out of memory")
|
|
63
|
+
_OOM_SPIN_MIN_LINES = 8
|
|
64
|
+
_OOM_SPIN_QUIET_SEC = 30.0
|
|
65
|
+
# Reciclagem do worker persistente: fragmentação/estado CUDA acumulam entre
|
|
66
|
+
# jobs — após N jobs o worker morre e o próximo load re-spawna fresco.
|
|
67
|
+
DEFAULT_WORKER_MAX_JOBS = 12 # 10 min sem progress → force abort
|
|
57
68
|
# Após progress pct≥1.0 / msg "done": o adapter já acabou o GPU work — só
|
|
58
69
|
# falta ``EVENT_DONE``. NÃO renovar o idle de 600s (era o hang de ~7-10 min
|
|
59
70
|
# com UI a 100% done e ``completed=0``). Grace curto para scrub+emit.
|
|
@@ -174,6 +185,7 @@ class _WorkerState:
|
|
|
174
185
|
# File handle do stderr do worker (aberto em _spawn, fechado em
|
|
175
186
|
# shutdown/_force_abort — antes ficava órfão e só era recuperado por GC).
|
|
176
187
|
log_fh: Any = None
|
|
188
|
+
jobs_served: int = 0
|
|
177
189
|
|
|
178
190
|
|
|
179
191
|
class SubprocessWorkerError(Exception):
|
|
@@ -199,6 +211,7 @@ class SubprocessWorkerPool:
|
|
|
199
211
|
post_done_timeout_sec: float | None = None,
|
|
200
212
|
ping_timeout_sec: float = DEFAULT_PING_TIMEOUT_SEC,
|
|
201
213
|
python_override: dict[str, str] | None = None,
|
|
214
|
+
worker_max_jobs: int | None = None,
|
|
202
215
|
) -> None:
|
|
203
216
|
self._spawn_fn = spawn_fn
|
|
204
217
|
self._log_path_fn = log_path_fn
|
|
@@ -218,6 +231,11 @@ class SubprocessWorkerPool:
|
|
|
218
231
|
if post_done_timeout_sec is not None
|
|
219
232
|
else _env_float("VRAMD_POST_DONE_TIMEOUT_SEC", DEFAULT_POST_DONE_TIMEOUT_SEC)
|
|
220
233
|
)
|
|
234
|
+
self._worker_max_jobs = int(
|
|
235
|
+
worker_max_jobs
|
|
236
|
+
if worker_max_jobs is not None
|
|
237
|
+
else _env_float("VRAMD_WORKER_MAX_JOBS", DEFAULT_WORKER_MAX_JOBS)
|
|
238
|
+
)
|
|
221
239
|
self._ping_timeout = float(ping_timeout_sec)
|
|
222
240
|
# Override do interpretador python por backend (testes / ambientes exóticos).
|
|
223
241
|
self._python_override: dict[str, str] = dict(python_override or {})
|
|
@@ -369,70 +387,99 @@ class SubprocessWorkerPool:
|
|
|
369
387
|
# estado interno (state["abort"] + emissor de progress).
|
|
370
388
|
serializable_request = {k: v for k, v in request.items() if not k.startswith("_") and not callable(v)}
|
|
371
389
|
self._safe_send(backend, state, CMD_GENERATE, request=serializable_request)
|
|
390
|
+
# Idle timeout por backend (RuntimeSpec.event_timeout_sec) — senão default.
|
|
391
|
+
rt = self._runtimes.get(backend)
|
|
392
|
+
ev_timeout = float(getattr(rt, "event_timeout_sec", None) or 0.0) or self._event_timeout
|
|
372
393
|
# Idle timeout: renovado a cada progress (generate longo OK).
|
|
373
|
-
idle_deadline = time.monotonic() +
|
|
394
|
+
idle_deadline = time.monotonic() + ev_timeout
|
|
374
395
|
abort_sent = False
|
|
375
396
|
abort_deadline: float | None = None
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
397
|
+
oom_lines = 0
|
|
398
|
+
oom_offset = self._worker_log_size(state)
|
|
399
|
+
last_progress = time.monotonic()
|
|
400
|
+
try:
|
|
401
|
+
while True:
|
|
402
|
+
now = time.monotonic()
|
|
403
|
+
if now >= idle_deadline:
|
|
404
|
+
self._force_abort(state, backend)
|
|
405
|
+
raise SubprocessWorkerError(f"{backend}: worker wedged — idle {ev_timeout:.0f}s sem progress")
|
|
406
|
+
# Poll cooperativo: se o caller pediu abort, enviar ao worker.
|
|
407
|
+
if not abort_sent and should_abort and should_abort():
|
|
408
|
+
with contextlib.suppress(OSError, ValueError):
|
|
409
|
+
# Worker pode ter morrido entretanto — o idle/poll
|
|
410
|
+
# check a seguir apanha-o; não deixar BrokenPipe
|
|
411
|
+
# saltar daqui.
|
|
412
|
+
from vramd.worker.protocol import send_cmd
|
|
413
|
+
|
|
414
|
+
send_cmd(state.proc.stdin, CMD_ABORT) # type: ignore[union-attr]
|
|
415
|
+
abort_sent = True
|
|
416
|
+
abort_deadline = now + self._abort_timeout
|
|
417
|
+
_logger.info(
|
|
418
|
+
f"[vramd] worker {backend}: abort pedido — SIGTERM se sem done"
|
|
419
|
+
f" em {self._abort_timeout:.0f}s"
|
|
420
|
+
)
|
|
421
|
+
# Escalação: abort cooperativo ignorado (text3d mid image_to_3d).
|
|
422
|
+
if abort_sent and abort_deadline is not None and now >= abort_deadline:
|
|
423
|
+
self._force_abort(state, backend)
|
|
424
|
+
raise SubprocessWorkerError(
|
|
425
|
+
f"{backend}: abort timeout ({self._abort_timeout:.0f}s) — worker forçado (SIGTERM)"
|
|
426
|
+
)
|
|
427
|
+
wait = min(idle_deadline - now, 1.0)
|
|
428
|
+
if abort_deadline is not None:
|
|
429
|
+
wait = min(wait, max(0.05, abort_deadline - now))
|
|
430
|
+
event = self._read_event_with_timeout(state, timeout=wait)
|
|
431
|
+
if event is None:
|
|
432
|
+
# Silêncio ≠ morte: só falha se o processo já saiu.
|
|
433
|
+
if state.proc is None or state.proc.poll() is not None:
|
|
434
|
+
state.loaded = False
|
|
435
|
+
raise SubprocessWorkerError(f"{backend}: worker fechou stdout mid-generate")
|
|
436
|
+
# Watcher de OOM-spin: o allocator em retry não emite
|
|
437
|
+
# eventos — as linhas de OOM crescem no stderr.
|
|
438
|
+
n, oom_offset = self._scan_worker_oom(state, oom_offset)
|
|
439
|
+
oom_lines += n
|
|
440
|
+
if oom_lines >= _OOM_SPIN_MIN_LINES and (now - last_progress) >= _OOM_SPIN_QUIET_SEC:
|
|
441
|
+
self._force_abort(state, backend)
|
|
442
|
+
raise SubprocessWorkerError(
|
|
443
|
+
f"{backend}: worker wedged — OOM-spin ({oom_lines} allocs, "
|
|
444
|
+
f"{now - last_progress:.0f}s sem progress)"
|
|
445
|
+
)
|
|
446
|
+
continue
|
|
447
|
+
ev = event["event"]
|
|
448
|
+
if ev == "progress":
|
|
449
|
+
last_progress = time.monotonic()
|
|
450
|
+
pct, msg = event.get("pct"), event.get("msg")
|
|
451
|
+
# Progress terminal NÃO renova os 600s — senão um DONE
|
|
452
|
+
# perdido/atrasado segura inflight ~10 min com UI a 100%.
|
|
453
|
+
if _progress_is_terminal(pct, msg):
|
|
454
|
+
idle_deadline = time.monotonic() + self._post_done_timeout
|
|
455
|
+
else:
|
|
456
|
+
idle_deadline = time.monotonic() + ev_timeout
|
|
457
|
+
if on_progress:
|
|
458
|
+
with contextlib.suppress(Exception):
|
|
459
|
+
on_progress(pct, msg)
|
|
460
|
+
continue
|
|
461
|
+
if ev == "vram_budget":
|
|
462
|
+
state.vram_mib = event.get("vram_mib", state.vram_mib)
|
|
463
|
+
continue
|
|
464
|
+
if ev == EVENT_DONE:
|
|
465
|
+
result = event.get("result", {})
|
|
466
|
+
return result
|
|
467
|
+
if ev == EVENT_ERROR:
|
|
468
|
+
raise SubprocessWorkerError(
|
|
469
|
+
f"{backend}: generate erro — {event.get('error')} ({event.get('error_code')})"
|
|
470
|
+
)
|
|
471
|
+
# Evento inesperado (pong, ready, unloaded): ignorar.
|
|
472
|
+
finally:
|
|
473
|
+
# Reciclagem: worker persistente acumula fragmentação/estado
|
|
474
|
+
# CUDA entre jobs (OOM residual, alocador partido) — após N
|
|
475
|
+
# jobs morre e o próximo load re-spawna fresco.
|
|
476
|
+
state.jobs_served += 1
|
|
477
|
+
if state.jobs_served >= self._worker_max_jobs and state.proc is not None and state.proc.poll() is None:
|
|
393
478
|
_logger.info(
|
|
394
|
-
f"[vramd] worker {backend}:
|
|
479
|
+
f"[vramd] worker {backend}: {state.jobs_served} jobs servidos — a reciclar "
|
|
480
|
+
f"(próximo load re-spawna)"
|
|
395
481
|
)
|
|
396
|
-
# Escalação: abort cooperativo ignorado (text3d mid image_to_3d).
|
|
397
|
-
if abort_sent and abort_deadline is not None and now >= abort_deadline:
|
|
398
482
|
self._force_abort(state, backend)
|
|
399
|
-
raise SubprocessWorkerError(
|
|
400
|
-
f"{backend}: abort timeout ({self._abort_timeout:.0f}s) — worker forçado (SIGTERM)"
|
|
401
|
-
)
|
|
402
|
-
wait = min(idle_deadline - now, 1.0)
|
|
403
|
-
if abort_deadline is not None:
|
|
404
|
-
wait = min(wait, max(0.05, abort_deadline - now))
|
|
405
|
-
event = self._read_event_with_timeout(state, timeout=wait)
|
|
406
|
-
if event is None:
|
|
407
|
-
# Silêncio ≠ morte: só falha se o processo já saiu.
|
|
408
|
-
if state.proc is None or state.proc.poll() is not None:
|
|
409
|
-
state.loaded = False
|
|
410
|
-
raise SubprocessWorkerError(f"{backend}: worker fechou stdout mid-generate")
|
|
411
|
-
continue
|
|
412
|
-
ev = event["event"]
|
|
413
|
-
if ev == "progress":
|
|
414
|
-
pct, msg = event.get("pct"), event.get("msg")
|
|
415
|
-
# Progress terminal NÃO renova os 600s — senão um DONE
|
|
416
|
-
# perdido/atrasado segura inflight ~10 min com UI a 100%.
|
|
417
|
-
if _progress_is_terminal(pct, msg):
|
|
418
|
-
idle_deadline = time.monotonic() + self._post_done_timeout
|
|
419
|
-
else:
|
|
420
|
-
idle_deadline = time.monotonic() + self._event_timeout
|
|
421
|
-
if on_progress:
|
|
422
|
-
with contextlib.suppress(Exception):
|
|
423
|
-
on_progress(pct, msg)
|
|
424
|
-
continue
|
|
425
|
-
if ev == "vram_budget":
|
|
426
|
-
state.vram_mib = event.get("vram_mib", state.vram_mib)
|
|
427
|
-
continue
|
|
428
|
-
if ev == EVENT_DONE:
|
|
429
|
-
result = event.get("result", {})
|
|
430
|
-
return result
|
|
431
|
-
if ev == EVENT_ERROR:
|
|
432
|
-
raise SubprocessWorkerError(
|
|
433
|
-
f"{backend}: generate erro — {event.get('error')} ({event.get('error_code')})"
|
|
434
|
-
)
|
|
435
|
-
# Evento inesperado (pong, ready, unloaded): ignorar.
|
|
436
483
|
|
|
437
484
|
def unload(self, backend: str) -> bool:
|
|
438
485
|
"""Manda o worker descarregar o modelo (worker persiste vivo)."""
|
|
@@ -730,6 +777,36 @@ class SubprocessWorkerPool:
|
|
|
730
777
|
_logger.info(f"[vramd] worker {state.backend}: evento inesperado {ev} (à espera de {expected})")
|
|
731
778
|
return None
|
|
732
779
|
|
|
780
|
+
def _worker_log_size(self, state: _WorkerState) -> int:
|
|
781
|
+
"""Tamanho actual do stderr do worker (0 se ainda não existe)."""
|
|
782
|
+
if state.log_path is None:
|
|
783
|
+
return 0
|
|
784
|
+
try:
|
|
785
|
+
return state.log_path.stat().st_size
|
|
786
|
+
except OSError:
|
|
787
|
+
return 0
|
|
788
|
+
|
|
789
|
+
def _scan_worker_oom(self, state: _WorkerState, offset: int) -> tuple[int, int]:
|
|
790
|
+
"""Linhas OOM novas no stderr do worker desde ``offset``.
|
|
791
|
+
|
|
792
|
+
Returns:
|
|
793
|
+
``(contagem de padrões OOM no trecho novo, novo offset)``.
|
|
794
|
+
"""
|
|
795
|
+
path = state.log_path
|
|
796
|
+
if path is None:
|
|
797
|
+
return 0, offset
|
|
798
|
+
try:
|
|
799
|
+
size = path.stat().st_size
|
|
800
|
+
if size <= offset:
|
|
801
|
+
return 0, offset
|
|
802
|
+
with open(path, "rb") as fh:
|
|
803
|
+
fh.seek(offset)
|
|
804
|
+
chunk = fh.read(size - offset)
|
|
805
|
+
except OSError:
|
|
806
|
+
return 0, offset
|
|
807
|
+
text = chunk.decode("utf-8", errors="ignore")
|
|
808
|
+
return sum(text.count(pat) for pat in _OOM_SPIN_PATTERNS), size
|
|
809
|
+
|
|
733
810
|
def _force_abort(self, state: _WorkerState, backend: str) -> None:
|
|
734
811
|
"""Abort cooperativo já falhou: SIGTERM e re-spawn limpo."""
|
|
735
812
|
_logger.warn(f"[vramd] worker {backend}: abort forçado (SIGTERM)")
|
|
@@ -273,6 +273,57 @@ class TestRefCounting:
|
|
|
273
273
|
assert eff == inference_headroom_mib(650)
|
|
274
274
|
|
|
275
275
|
|
|
276
|
+
class TestMeasuredFootprintMemEffDiscount:
|
|
277
|
+
"""Regressão 0.3.4: o 0.65 NÃO pode descontar uma activação medida em mem-eff.
|
|
278
|
+
|
|
279
|
+
A calibração da 6 GB foi feita COM memory_efficient (load_kwargs no
|
|
280
|
+
peak_profile) — a activação medida já reflete o modo. Descontar de novo
|
|
281
|
+
subdimensionava o admit em ~1 GiB e admitia jobs condenados ao OOM-spin.
|
|
282
|
+
"""
|
|
283
|
+
|
|
284
|
+
@staticmethod
|
|
285
|
+
def _mgr(desc: BackendDescriptor) -> BackendManager:
|
|
286
|
+
registry = Registry(descriptors={"paint": desc})
|
|
287
|
+
registry._adapter_instances["paint"] = MockAdapter(name="paint")
|
|
288
|
+
return BackendManager(registry, query_free_mib=lambda: 99999, clear_vram=lambda: None)
|
|
289
|
+
|
|
290
|
+
@staticmethod
|
|
291
|
+
def _desc(mem_eff_measured: bool, *, admit_peak: int | None = None) -> BackendDescriptor:
|
|
292
|
+
vram: dict = {"weights_gib": 1.7, "activation_gib": 3.9, "context_gib": 0.0}
|
|
293
|
+
if admit_peak:
|
|
294
|
+
vram["admit_peak_mib"] = admit_peak
|
|
295
|
+
return BackendDescriptor(
|
|
296
|
+
name="paint",
|
|
297
|
+
adapter="mock",
|
|
298
|
+
vram_mib=6000,
|
|
299
|
+
priority=40,
|
|
300
|
+
vram=vram,
|
|
301
|
+
peak_profile={"quant_mode": "sdnq-fp8", "load_kwargs": {"memory_efficient": mem_eff_measured}},
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
def test_measured_mem_eff_activation_not_discounted_twice(self) -> None:
|
|
305
|
+
mgr = self._mgr(self._desc(mem_eff_measured=True))
|
|
306
|
+
_w, activation = mgr.footprint_parts_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True)
|
|
307
|
+
assert activation == int(3.9 * 1024) # intacta — já foi medida em mem-eff
|
|
308
|
+
assert activation > int(3.9 * 1024 * 0.65) + 100 # sem duplo desconto
|
|
309
|
+
|
|
310
|
+
def test_measured_fp16_activation_gets_mem_eff_discount(self) -> None:
|
|
311
|
+
mgr = self._mgr(self._desc(mem_eff_measured=False))
|
|
312
|
+
_w, activation = mgr.footprint_parts_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True)
|
|
313
|
+
assert activation == max(512, int(int(3.9 * 1024) * 0.65))
|
|
314
|
+
|
|
315
|
+
def test_admit_peak_mib_wins_over_analytic_sum(self) -> None:
|
|
316
|
+
mgr = self._mgr(self._desc(mem_eff_measured=True, admit_peak=6144))
|
|
317
|
+
assert mgr.peak_vram_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True) == 6144
|
|
318
|
+
|
|
319
|
+
def test_without_admit_peak_uses_analytic_sum(self) -> None:
|
|
320
|
+
from vramd.vram_planner import peak_vram_mib as compute_peak_mib
|
|
321
|
+
|
|
322
|
+
mgr = self._mgr(self._desc(mem_eff_measured=True))
|
|
323
|
+
w, a = mgr.footprint_parts_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True)
|
|
324
|
+
assert mgr.peak_vram_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True) == compute_peak_mib(w, a)
|
|
325
|
+
|
|
326
|
+
|
|
276
327
|
class TestErrorRecovery:
|
|
277
328
|
"""Em erro de geração, o backend é descarregado para recovery."""
|
|
278
329
|
|
|
@@ -492,7 +492,92 @@ class TestBufferedStdoutLine:
|
|
|
492
492
|
import time
|
|
493
493
|
|
|
494
494
|
t0 = time.monotonic()
|
|
495
|
-
with pytest.raises(SubprocessWorkerError, match="
|
|
495
|
+
with pytest.raises(SubprocessWorkerError, match="worker wedged"):
|
|
496
496
|
pool.generate("paint3d", {})
|
|
497
497
|
elapsed = time.monotonic() - t0
|
|
498
498
|
assert elapsed < 5.0, f"esperava post_done curto, demorou {elapsed:.1f}s"
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
class TestOomWatcher:
|
|
502
|
+
"""Watcher de OOM-spin: conta só o que cresce no stderr durante o job."""
|
|
503
|
+
|
|
504
|
+
def test_scan_counts_only_new_lines(self, tmp_path):
|
|
505
|
+
from vramd.subprocess_pool import _WorkerState
|
|
506
|
+
|
|
507
|
+
pool = _make_pool(FakePopen(), log_path_fn=lambda b: tmp_path / "w.log")
|
|
508
|
+
log = tmp_path / "w.log"
|
|
509
|
+
log.write_text("expandable_segments: memory mapping failed with OOM\n" * 3, encoding="utf-8")
|
|
510
|
+
state = _WorkerState(backend="paint3d", log_path=log)
|
|
511
|
+
|
|
512
|
+
n, off = pool._scan_worker_oom(state, 0)
|
|
513
|
+
assert n == 3
|
|
514
|
+
assert off == log.stat().st_size
|
|
515
|
+
|
|
516
|
+
# Re-scan do mesmo offset: nada novo.
|
|
517
|
+
assert pool._scan_worker_oom(state, off) == (0, off)
|
|
518
|
+
|
|
519
|
+
with open(log, "a", encoding="utf-8") as fh:
|
|
520
|
+
fh.write("linha normal\nCUDA out of memory\n")
|
|
521
|
+
n2, off2 = pool._scan_worker_oom(state, off)
|
|
522
|
+
assert n2 == 1 # só o padrão novo conta
|
|
523
|
+
assert off2 == log.stat().st_size
|
|
524
|
+
|
|
525
|
+
def test_wedge_aborts_with_classified_error(self, tmp_path, monkeypatch):
|
|
526
|
+
"""OOM-spin (8+ linhas novas, sem progresso) → abort em segundos."""
|
|
527
|
+
import threading
|
|
528
|
+
|
|
529
|
+
import vramd.subprocess_pool as sp
|
|
530
|
+
|
|
531
|
+
monkeypatch.setattr(sp, "_OOM_SPIN_QUIET_SEC", 2.0)
|
|
532
|
+
fake = FakePopen()
|
|
533
|
+
pool = _make_pool(
|
|
534
|
+
fake,
|
|
535
|
+
event_timeout_sec=60.0,
|
|
536
|
+
log_path_fn=lambda b: tmp_path / "worker.log",
|
|
537
|
+
)
|
|
538
|
+
fake.push_events('{"event": "ready", "vram_mib": 1300}')
|
|
539
|
+
pool.load("paint3d", "paint3d", {})
|
|
540
|
+
log = tmp_path / "worker.log"
|
|
541
|
+
|
|
542
|
+
def writer() -> None:
|
|
543
|
+
import time
|
|
544
|
+
|
|
545
|
+
time.sleep(1.0)
|
|
546
|
+
log.write_text("expandable_segments: memory mapping failed with OOM\n" * 9, encoding="utf-8")
|
|
547
|
+
|
|
548
|
+
t = threading.Thread(target=writer)
|
|
549
|
+
t.start()
|
|
550
|
+
try:
|
|
551
|
+
with pytest.raises(SubprocessWorkerError, match="worker wedged — OOM-spin"):
|
|
552
|
+
pool.generate("paint3d", {})
|
|
553
|
+
finally:
|
|
554
|
+
t.join()
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
class TestWorkerDeadPredicate:
|
|
558
|
+
""" "worker wedged" entra no predicado de requeue (worker morto transitório)."""
|
|
559
|
+
|
|
560
|
+
def test_wedge_messages_trigger_requeue(self) -> None:
|
|
561
|
+
from vramd.dispatcher import _is_worker_dead
|
|
562
|
+
|
|
563
|
+
assert _is_worker_dead({"error": "paint3d: worker wedged — OOM-spin (12 allocs, 40s sem progress)"})
|
|
564
|
+
assert _is_worker_dead({"error": "paint3d: worker wedged — idle 600s sem progress"})
|
|
565
|
+
assert not _is_worker_dead({"error": "erro genérico de geração"})
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
class TestWorkerRecycling:
|
|
569
|
+
"""Worker persistente recicla após N jobs (fragmentação/estado não acumula)."""
|
|
570
|
+
|
|
571
|
+
def test_recycles_after_max_jobs(self):
|
|
572
|
+
fake = FakePopen()
|
|
573
|
+
pool = _make_pool(fake, worker_max_jobs=3)
|
|
574
|
+
fake.push_events('{"event": "ready", "vram_mib": 1300}')
|
|
575
|
+
pool.load("paint3d", "paint3d", {})
|
|
576
|
+
state = pool._workers["paint3d"]
|
|
577
|
+
for _i in range(3):
|
|
578
|
+
fake.push_events('{"event": "progress", "pct": 0.5}')
|
|
579
|
+
fake.push_events('{"event": "done", "result": {"status": "ok"}}')
|
|
580
|
+
out = pool.generate("paint3d", {})
|
|
581
|
+
assert out["status"] == "ok"
|
|
582
|
+
assert state.jobs_served == 3
|
|
583
|
+
assert state.proc is None # reciclado: próximo load re-spawna
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Consistência de versão: o literal __version__ acompanha o pyproject."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import vramd
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _pyproject_version() -> str:
|
|
12
|
+
pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
|
13
|
+
match = re.search(r'^version\s*=\s*"([^"]+)"', pyproject.read_text(encoding="utf-8"), re.MULTILINE)
|
|
14
|
+
assert match, "pyproject.toml sem version"
|
|
15
|
+
return match.group(1)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TestVersion:
|
|
19
|
+
def test_dunder_version_matches_pyproject(self) -> None:
|
|
20
|
+
# 0.3.2 saiu com __version__="0.3.1": `vramd --version` mentia.
|
|
21
|
+
assert vramd.__version__ == _pyproject_version()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|