vramd 0.3.2__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {vramd-0.3.2/src/vramd.egg-info → vramd-0.3.4}/PKG-INFO +1 -1
  2. {vramd-0.3.2 → vramd-0.3.4}/pyproject.toml +1 -1
  3. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/__init__.py +1 -1
  4. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/backend_manager.py +18 -1
  5. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/dispatcher.py +7 -1
  6. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/server.py +7 -0
  7. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/subprocess_pool.py +136 -59
  8. {vramd-0.3.2 → vramd-0.3.4/src/vramd.egg-info}/PKG-INFO +1 -1
  9. {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/SOURCES.txt +1 -0
  10. {vramd-0.3.2 → vramd-0.3.4}/tests/test_backend_manager.py +51 -0
  11. {vramd-0.3.2 → vramd-0.3.4}/tests/test_subprocess_pool.py +86 -1
  12. vramd-0.3.4/tests/test_version.py +21 -0
  13. {vramd-0.3.2 → vramd-0.3.4}/LICENSE +0 -0
  14. {vramd-0.3.2 → vramd-0.3.4}/README.md +0 -0
  15. {vramd-0.3.2 → vramd-0.3.4}/setup.cfg +0 -0
  16. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/__main__.py +0 -0
  17. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/adapter.py +0 -0
  18. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/__init__.py +0 -0
  19. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/analysis.py +0 -0
  20. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/compare.py +0 -0
  21. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/emit.py +0 -0
  22. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/hw_auto.py +0 -0
  23. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/runner.py +0 -0
  24. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/sampler.py +0 -0
  25. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/calibrate/serde.py +0 -0
  26. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/cli.py +0 -0
  27. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/client.py +0 -0
  28. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/data/backends.yaml +0 -0
  29. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/data/test-image.png +0 -0
  30. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/data/test-mesh.glb +0 -0
  31. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/errors.py +0 -0
  32. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/footprints.py +0 -0
  33. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/gpu.py +0 -0
  34. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/hooks.py +0 -0
  35. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/idle_evictor.py +0 -0
  36. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/job_queue.py +0 -0
  37. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/learn.py +0 -0
  38. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/logging.py +0 -0
  39. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/mcp_server.py +0 -0
  40. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/process_guard.py +0 -0
  41. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/protocol.py +0 -0
  42. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/py.typed +0 -0
  43. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/registry.py +0 -0
  44. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/scheduler.py +0 -0
  45. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/stats.py +0 -0
  46. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/toolchain.py +0 -0
  47. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/top.py +0 -0
  48. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/vram_planner.py +0 -0
  49. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/worker/__init__.py +0 -0
  50. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/worker/adapter.py +0 -0
  51. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/worker/protocol.py +0 -0
  52. {vramd-0.3.2 → vramd-0.3.4}/src/vramd/worker/serve.py +0 -0
  53. {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/dependency_links.txt +0 -0
  54. {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/entry_points.txt +0 -0
  55. {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/requires.txt +0 -0
  56. {vramd-0.3.2 → vramd-0.3.4}/src/vramd.egg-info/top_level.txt +0 -0
  57. {vramd-0.3.2 → vramd-0.3.4}/tests/test_abort_progress.py +0 -0
  58. {vramd-0.3.2 → vramd-0.3.4}/tests/test_backend_manager_hybrid.py +0 -0
  59. {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_analysis.py +0 -0
  60. {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_cli.py +0 -0
  61. {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_emit_compare.py +0 -0
  62. {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_hw_auto.py +0 -0
  63. {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_runner.py +0 -0
  64. {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_sampler.py +0 -0
  65. {vramd-0.3.2 → vramd-0.3.4}/tests/test_calibrate_serde.py +0 -0
  66. {vramd-0.3.2 → vramd-0.3.4}/tests/test_cli_debug_stats_bench.py +0 -0
  67. {vramd-0.3.2 → vramd-0.3.4}/tests/test_cli_start.py +0 -0
  68. {vramd-0.3.2 → vramd-0.3.4}/tests/test_gpu_ids_load.py +0 -0
  69. {vramd-0.3.2 → vramd-0.3.4}/tests/test_hooks.py +0 -0
  70. {vramd-0.3.2 → vramd-0.3.4}/tests/test_idle_evictor.py +0 -0
  71. {vramd-0.3.2 → vramd-0.3.4}/tests/test_job_queue.py +0 -0
  72. {vramd-0.3.2 → vramd-0.3.4}/tests/test_job_wal.py +0 -0
  73. {vramd-0.3.2 → vramd-0.3.4}/tests/test_learn.py +0 -0
  74. {vramd-0.3.2 → vramd-0.3.4}/tests/test_learn_integration.py +0 -0
  75. {vramd-0.3.2 → vramd-0.3.4}/tests/test_mcp_server.py +0 -0
  76. {vramd-0.3.2 → vramd-0.3.4}/tests/test_peak_profile.py +0 -0
  77. {vramd-0.3.2 → vramd-0.3.4}/tests/test_phase_progress.py +0 -0
  78. {vramd-0.3.2 → vramd-0.3.4}/tests/test_process_guard.py +0 -0
  79. {vramd-0.3.2 → vramd-0.3.4}/tests/test_protocol_env.py +0 -0
  80. {vramd-0.3.2 → vramd-0.3.4}/tests/test_queue_metrics_eta.py +0 -0
  81. {vramd-0.3.2 → vramd-0.3.4}/tests/test_registry.py +0 -0
  82. {vramd-0.3.2 → vramd-0.3.4}/tests/test_registry_layers.py +0 -0
  83. {vramd-0.3.2 → vramd-0.3.4}/tests/test_respawn.py +0 -0
  84. {vramd-0.3.2 → vramd-0.3.4}/tests/test_runtime_spawn.py +0 -0
  85. {vramd-0.3.2 → vramd-0.3.4}/tests/test_scheduler.py +0 -0
  86. {vramd-0.3.2 → vramd-0.3.4}/tests/test_server.py +0 -0
  87. {vramd-0.3.2 → vramd-0.3.4}/tests/test_stats.py +0 -0
  88. {vramd-0.3.2 → vramd-0.3.4}/tests/test_subprocess_integration.py +0 -0
  89. {vramd-0.3.2 → vramd-0.3.4}/tests/test_subprocess_per_backend_e2e.py +0 -0
  90. {vramd-0.3.2 → vramd-0.3.4}/tests/test_toolchain.py +0 -0
  91. {vramd-0.3.2 → vramd-0.3.4}/tests/test_top.py +0 -0
  92. {vramd-0.3.2 → vramd-0.3.4}/tests/test_unload_frees_vram.py +0 -0
  93. {vramd-0.3.2 → vramd-0.3.4}/tests/test_vram_admit.py +0 -0
  94. {vramd-0.3.2 → vramd-0.3.4}/tests/test_vram_planner.py +0 -0
  95. {vramd-0.3.2 → vramd-0.3.4}/tests/test_vramd_coverage_100.py +0 -0
  96. {vramd-0.3.2 → vramd-0.3.4}/tests/test_worker.py +0 -0
  97. {vramd-0.3.2 → vramd-0.3.4}/tests/test_zero_vram.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: vramd
3
- Version: 0.3.2
3
+ Version: 0.3.4
4
4
  Summary: VRAM admission control for generative inference on consumer GPUs
5
5
  Author: Maikeu Locatelli
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "vramd"
7
- version = "0.3.2"
7
+ version = "0.3.4"
8
8
  description = "VRAM admission control for generative inference on consumer GPUs"
9
9
  readme = "README.md"
10
10
  authors = [{ name = "Maikeu Locatelli" }]
@@ -19,5 +19,5 @@ adivinhar: o número que o admit usa vem do driver, não de uma estimativa.
19
19
 
20
20
  from __future__ import annotations
21
21
 
22
- __version__ = "0.3.1"
22
+ __version__ = "0.3.4"
23
23
  __all__ = ["__version__"]
@@ -692,7 +692,16 @@ class BackendManager:
692
692
  measured = self._measured_parts_mib(desc, quant_mode=quant_mode)
693
693
  if measured is not None:
694
694
  weights_measured, activation_measured = measured
695
- if memory_efficient and not group_offload:
695
+ measured_mem_eff = bool(
696
+ ((getattr(desc, "peak_profile", None) or {}).get("load_kwargs") or {}).get("memory_efficient")
697
+ )
698
+ # O desconto 0.65 só vale quando a medição foi feita SEM mem-eff
699
+ # (realidade FP16) e o request agora pede o modo económico. Se a
700
+ # calibração já foi medida COM memory_efficient (caso paint3d 6g:
701
+ # load_kwargs.memory_efficient=true), a activação medida JÁ reflete
702
+ # o modo — descontar de novo subdimensionava o admit em ~1 GiB e
703
+ # admitia jobs que estouravam (OOM-spin do worker).
704
+ if memory_efficient and not group_offload and not measured_mem_eff:
696
705
  activation_measured = max(512, int(activation_measured * _MEMORY_EFFICIENT_ACTIVATION_FACTOR))
697
706
  return weights_measured, activation_measured
698
707
 
@@ -753,6 +762,14 @@ class BackendManager:
753
762
  footprint_key: str | None = None,
754
763
  ) -> int:
755
764
  """Pico = pesos(quant) + activação de inferência + safety."""
765
+ with contextlib.suppress(KeyError):
766
+ vram = getattr(self._registry.descriptor(name), "vram", None) or {}
767
+ admit_peak = vram.get("admit_peak_mib")
768
+ if admit_peak:
769
+ # Recomendação do calibrador: pico REAL medido (pesos +
770
+ # activação + safety). Vence a soma analítica — que pode
771
+ # descontar mem-eff duas vezes e admitir jobs condenados.
772
+ return int(admit_peak)
756
773
  weights, activation = self.footprint_parts_mib(
757
774
  name,
758
775
  quant_mode=quant_mode,
@@ -38,7 +38,13 @@ def _is_worker_dead(result: dict[str, Any]) -> bool:
38
38
  if result.get("error_code") == P.ERR_WORKER_DEAD:
39
39
  return True
40
40
  err = str(result.get("error") or "").lower()
41
- return "não está vivo" in err or "nao esta vivo" in err or "worker fechou stdout" in err or "eof no load" in err
41
+ return (
42
+ "não está vivo" in err
43
+ or "nao esta vivo" in err
44
+ or "worker fechou stdout" in err
45
+ or "eof no load" in err
46
+ or "worker wedged" in err
47
+ )
42
48
 
43
49
 
44
50
  def _vram_retry_worthwhile(result: dict[str, Any], total_mib: int | None) -> bool:
@@ -420,6 +420,13 @@ class VramdServer:
420
420
  if "cancel" in err_txt:
421
421
  out.setdefault("error_code", P.ERR_CANCELLED)
422
422
  out.setdefault("hint", "Job cancelado (queued ou durante generate).")
423
+ elif "oom" in err_txt:
424
+ out.setdefault("error_code", P.ERR_GENERATE_FAILED)
425
+ out.setdefault(
426
+ "hint",
427
+ "OOM/spin no worker — vê o log vramd-worker-<backend>.log; "
428
+ "considera reduzir tamanho/qualidade do pedido.",
429
+ )
423
430
  elif "timeout" in err_txt:
424
431
  out.setdefault("error_code", P.ERR_TIMEOUT)
425
432
  out.setdefault("hint", "Aumenta timeout do cliente ou inspecciona vramd queue.")
@@ -53,7 +53,18 @@ _logger = Logger()
53
53
 
54
54
  # Idle entre eventos de progresso (reset a cada progress). Hunyuan pode
55
55
  # demorar muitos minutos no total — NÃO é wall-clock do generate inteiro.
56
- DEFAULT_EVENT_TIMEOUT_SEC = 600.0 # 10 min sem progress → force abort
56
+ DEFAULT_EVENT_TIMEOUT_SEC = 600.0
57
+
58
+ # Watcher de OOM-spin: o allocator CUDA em retry infinito não emite eventos —
59
+ # cresce no stderr do worker (~0.7 linhas/s de "expandable_segments: memory
60
+ # mapping failed"). 8 linhas + 30s sem progress denunciam o wedge em ~40s
61
+ # em vez dos 600s do idle-timeout (que depois virava "timeout do cliente").
62
+ _OOM_SPIN_PATTERNS = ("expandable_segments: memory mapping failed", "CUDA out of memory")
63
+ _OOM_SPIN_MIN_LINES = 8
64
+ _OOM_SPIN_QUIET_SEC = 30.0
65
+ # Reciclagem do worker persistente: fragmentação/estado CUDA acumulam entre
66
+ # jobs — após N jobs o worker morre e o próximo load re-spawna fresco.
67
+ DEFAULT_WORKER_MAX_JOBS = 12 # 10 min sem progress → force abort
57
68
  # Após progress pct≥1.0 / msg "done": o adapter já acabou o GPU work — só
58
69
  # falta ``EVENT_DONE``. NÃO renovar o idle de 600s (era o hang de ~7-10 min
59
70
  # com UI a 100% done e ``completed=0``). Grace curto para scrub+emit.
@@ -174,6 +185,7 @@ class _WorkerState:
174
185
  # File handle do stderr do worker (aberto em _spawn, fechado em
175
186
  # shutdown/_force_abort — antes ficava órfão e só era recuperado por GC).
176
187
  log_fh: Any = None
188
+ jobs_served: int = 0
177
189
 
178
190
 
179
191
  class SubprocessWorkerError(Exception):
@@ -199,6 +211,7 @@ class SubprocessWorkerPool:
199
211
  post_done_timeout_sec: float | None = None,
200
212
  ping_timeout_sec: float = DEFAULT_PING_TIMEOUT_SEC,
201
213
  python_override: dict[str, str] | None = None,
214
+ worker_max_jobs: int | None = None,
202
215
  ) -> None:
203
216
  self._spawn_fn = spawn_fn
204
217
  self._log_path_fn = log_path_fn
@@ -218,6 +231,11 @@ class SubprocessWorkerPool:
218
231
  if post_done_timeout_sec is not None
219
232
  else _env_float("VRAMD_POST_DONE_TIMEOUT_SEC", DEFAULT_POST_DONE_TIMEOUT_SEC)
220
233
  )
234
+ self._worker_max_jobs = int(
235
+ worker_max_jobs
236
+ if worker_max_jobs is not None
237
+ else _env_float("VRAMD_WORKER_MAX_JOBS", DEFAULT_WORKER_MAX_JOBS)
238
+ )
221
239
  self._ping_timeout = float(ping_timeout_sec)
222
240
  # Override do interpretador python por backend (testes / ambientes exóticos).
223
241
  self._python_override: dict[str, str] = dict(python_override or {})
@@ -369,70 +387,99 @@ class SubprocessWorkerPool:
369
387
  # estado interno (state["abort"] + emissor de progress).
370
388
  serializable_request = {k: v for k, v in request.items() if not k.startswith("_") and not callable(v)}
371
389
  self._safe_send(backend, state, CMD_GENERATE, request=serializable_request)
390
+ # Idle timeout por backend (RuntimeSpec.event_timeout_sec) — senão default.
391
+ rt = self._runtimes.get(backend)
392
+ ev_timeout = float(getattr(rt, "event_timeout_sec", None) or 0.0) or self._event_timeout
372
393
  # Idle timeout: renovado a cada progress (generate longo OK).
373
- idle_deadline = time.monotonic() + self._event_timeout
394
+ idle_deadline = time.monotonic() + ev_timeout
374
395
  abort_sent = False
375
396
  abort_deadline: float | None = None
376
- while True:
377
- now = time.monotonic()
378
- if now >= idle_deadline:
379
- self._force_abort(state, backend)
380
- raise SubprocessWorkerError(
381
- f"{backend}: timeout idle ({self._event_timeout:.0f}s sem progress) no generate"
382
- )
383
- # Poll cooperativo: se o caller pediu abort, enviar ao worker.
384
- if not abort_sent and should_abort and should_abort():
385
- with contextlib.suppress(OSError, ValueError):
386
- # Worker pode ter morrido entretanto — o idle/poll check
387
- # a seguir apanha-o; não deixar BrokenPipe saltar daqui.
388
- from vramd.worker.protocol import send_cmd
389
-
390
- send_cmd(state.proc.stdin, CMD_ABORT) # type: ignore[union-attr]
391
- abort_sent = True
392
- abort_deadline = now + self._abort_timeout
397
+ oom_lines = 0
398
+ oom_offset = self._worker_log_size(state)
399
+ last_progress = time.monotonic()
400
+ try:
401
+ while True:
402
+ now = time.monotonic()
403
+ if now >= idle_deadline:
404
+ self._force_abort(state, backend)
405
+ raise SubprocessWorkerError(f"{backend}: worker wedged — idle {ev_timeout:.0f}s sem progress")
406
+ # Poll cooperativo: se o caller pediu abort, enviar ao worker.
407
+ if not abort_sent and should_abort and should_abort():
408
+ with contextlib.suppress(OSError, ValueError):
409
+ # Worker pode ter morrido entretanto — o idle/poll
410
+ # check a seguir apanha-o; não deixar BrokenPipe
411
+ # saltar daqui.
412
+ from vramd.worker.protocol import send_cmd
413
+
414
+ send_cmd(state.proc.stdin, CMD_ABORT) # type: ignore[union-attr]
415
+ abort_sent = True
416
+ abort_deadline = now + self._abort_timeout
417
+ _logger.info(
418
+ f"[vramd] worker {backend}: abort pedido — SIGTERM se sem done"
419
+ f" em {self._abort_timeout:.0f}s"
420
+ )
421
+ # Escalação: abort cooperativo ignorado (text3d mid image_to_3d).
422
+ if abort_sent and abort_deadline is not None and now >= abort_deadline:
423
+ self._force_abort(state, backend)
424
+ raise SubprocessWorkerError(
425
+ f"{backend}: abort timeout ({self._abort_timeout:.0f}s) — worker forçado (SIGTERM)"
426
+ )
427
+ wait = min(idle_deadline - now, 1.0)
428
+ if abort_deadline is not None:
429
+ wait = min(wait, max(0.05, abort_deadline - now))
430
+ event = self._read_event_with_timeout(state, timeout=wait)
431
+ if event is None:
432
+ # Silêncio ≠ morte: só falha se o processo já saiu.
433
+ if state.proc is None or state.proc.poll() is not None:
434
+ state.loaded = False
435
+ raise SubprocessWorkerError(f"{backend}: worker fechou stdout mid-generate")
436
+ # Watcher de OOM-spin: o allocator em retry não emite
437
+ # eventos — as linhas de OOM crescem no stderr.
438
+ n, oom_offset = self._scan_worker_oom(state, oom_offset)
439
+ oom_lines += n
440
+ if oom_lines >= _OOM_SPIN_MIN_LINES and (now - last_progress) >= _OOM_SPIN_QUIET_SEC:
441
+ self._force_abort(state, backend)
442
+ raise SubprocessWorkerError(
443
+ f"{backend}: worker wedged — OOM-spin ({oom_lines} allocs, "
444
+ f"{now - last_progress:.0f}s sem progress)"
445
+ )
446
+ continue
447
+ ev = event["event"]
448
+ if ev == "progress":
449
+ last_progress = time.monotonic()
450
+ pct, msg = event.get("pct"), event.get("msg")
451
+ # Progress terminal NÃO renova os 600s — senão um DONE
452
+ # perdido/atrasado segura inflight ~10 min com UI a 100%.
453
+ if _progress_is_terminal(pct, msg):
454
+ idle_deadline = time.monotonic() + self._post_done_timeout
455
+ else:
456
+ idle_deadline = time.monotonic() + ev_timeout
457
+ if on_progress:
458
+ with contextlib.suppress(Exception):
459
+ on_progress(pct, msg)
460
+ continue
461
+ if ev == "vram_budget":
462
+ state.vram_mib = event.get("vram_mib", state.vram_mib)
463
+ continue
464
+ if ev == EVENT_DONE:
465
+ result = event.get("result", {})
466
+ return result
467
+ if ev == EVENT_ERROR:
468
+ raise SubprocessWorkerError(
469
+ f"{backend}: generate erro — {event.get('error')} ({event.get('error_code')})"
470
+ )
471
+ # Evento inesperado (pong, ready, unloaded): ignorar.
472
+ finally:
473
+ # Reciclagem: worker persistente acumula fragmentação/estado
474
+ # CUDA entre jobs (OOM residual, alocador partido) — após N
475
+ # jobs morre e o próximo load re-spawna fresco.
476
+ state.jobs_served += 1
477
+ if state.jobs_served >= self._worker_max_jobs and state.proc is not None and state.proc.poll() is None:
393
478
  _logger.info(
394
- f"[vramd] worker {backend}: abort pedido — SIGTERM se sem done em {self._abort_timeout:.0f}s"
479
+ f"[vramd] worker {backend}: {state.jobs_served} jobs servidos — a reciclar "
480
+ f"(próximo load re-spawna)"
395
481
  )
396
- # Escalação: abort cooperativo ignorado (text3d mid image_to_3d).
397
- if abort_sent and abort_deadline is not None and now >= abort_deadline:
398
482
  self._force_abort(state, backend)
399
- raise SubprocessWorkerError(
400
- f"{backend}: abort timeout ({self._abort_timeout:.0f}s) — worker forçado (SIGTERM)"
401
- )
402
- wait = min(idle_deadline - now, 1.0)
403
- if abort_deadline is not None:
404
- wait = min(wait, max(0.05, abort_deadline - now))
405
- event = self._read_event_with_timeout(state, timeout=wait)
406
- if event is None:
407
- # Silêncio ≠ morte: só falha se o processo já saiu.
408
- if state.proc is None or state.proc.poll() is not None:
409
- state.loaded = False
410
- raise SubprocessWorkerError(f"{backend}: worker fechou stdout mid-generate")
411
- continue
412
- ev = event["event"]
413
- if ev == "progress":
414
- pct, msg = event.get("pct"), event.get("msg")
415
- # Progress terminal NÃO renova os 600s — senão um DONE
416
- # perdido/atrasado segura inflight ~10 min com UI a 100%.
417
- if _progress_is_terminal(pct, msg):
418
- idle_deadline = time.monotonic() + self._post_done_timeout
419
- else:
420
- idle_deadline = time.monotonic() + self._event_timeout
421
- if on_progress:
422
- with contextlib.suppress(Exception):
423
- on_progress(pct, msg)
424
- continue
425
- if ev == "vram_budget":
426
- state.vram_mib = event.get("vram_mib", state.vram_mib)
427
- continue
428
- if ev == EVENT_DONE:
429
- result = event.get("result", {})
430
- return result
431
- if ev == EVENT_ERROR:
432
- raise SubprocessWorkerError(
433
- f"{backend}: generate erro — {event.get('error')} ({event.get('error_code')})"
434
- )
435
- # Evento inesperado (pong, ready, unloaded): ignorar.
436
483
 
437
484
  def unload(self, backend: str) -> bool:
438
485
  """Manda o worker descarregar o modelo (worker persiste vivo)."""
@@ -730,6 +777,36 @@ class SubprocessWorkerPool:
730
777
  _logger.info(f"[vramd] worker {state.backend}: evento inesperado {ev} (à espera de {expected})")
731
778
  return None
732
779
 
780
+ def _worker_log_size(self, state: _WorkerState) -> int:
781
+ """Tamanho actual do stderr do worker (0 se ainda não existe)."""
782
+ if state.log_path is None:
783
+ return 0
784
+ try:
785
+ return state.log_path.stat().st_size
786
+ except OSError:
787
+ return 0
788
+
789
+ def _scan_worker_oom(self, state: _WorkerState, offset: int) -> tuple[int, int]:
790
+ """Linhas OOM novas no stderr do worker desde ``offset``.
791
+
792
+ Returns:
793
+ ``(contagem de padrões OOM no trecho novo, novo offset)``.
794
+ """
795
+ path = state.log_path
796
+ if path is None:
797
+ return 0, offset
798
+ try:
799
+ size = path.stat().st_size
800
+ if size <= offset:
801
+ return 0, offset
802
+ with open(path, "rb") as fh:
803
+ fh.seek(offset)
804
+ chunk = fh.read(size - offset)
805
+ except OSError:
806
+ return 0, offset
807
+ text = chunk.decode("utf-8", errors="ignore")
808
+ return sum(text.count(pat) for pat in _OOM_SPIN_PATTERNS), size
809
+
733
810
  def _force_abort(self, state: _WorkerState, backend: str) -> None:
734
811
  """Abort cooperativo já falhou: SIGTERM e re-spawn limpo."""
735
812
  _logger.warn(f"[vramd] worker {backend}: abort forçado (SIGTERM)")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: vramd
3
- Version: 0.3.2
3
+ Version: 0.3.4
4
4
  Summary: VRAM admission control for generative inference on consumer GPUs
5
5
  Author: Maikeu Locatelli
6
6
  License-Expression: MIT
@@ -87,6 +87,7 @@ tests/test_subprocess_pool.py
87
87
  tests/test_toolchain.py
88
88
  tests/test_top.py
89
89
  tests/test_unload_frees_vram.py
90
+ tests/test_version.py
90
91
  tests/test_vram_admit.py
91
92
  tests/test_vram_planner.py
92
93
  tests/test_vramd_coverage_100.py
@@ -273,6 +273,57 @@ class TestRefCounting:
273
273
  assert eff == inference_headroom_mib(650)
274
274
 
275
275
 
276
+ class TestMeasuredFootprintMemEffDiscount:
277
+ """Regressão 0.3.4: o 0.65 NÃO pode descontar uma activação medida em mem-eff.
278
+
279
+ A calibração da 6 GB foi feita COM memory_efficient (load_kwargs no
280
+ peak_profile) — a activação medida já reflete o modo. Descontar de novo
281
+ subdimensionava o admit em ~1 GiB e admitia jobs condenados ao OOM-spin.
282
+ """
283
+
284
+ @staticmethod
285
+ def _mgr(desc: BackendDescriptor) -> BackendManager:
286
+ registry = Registry(descriptors={"paint": desc})
287
+ registry._adapter_instances["paint"] = MockAdapter(name="paint")
288
+ return BackendManager(registry, query_free_mib=lambda: 99999, clear_vram=lambda: None)
289
+
290
+ @staticmethod
291
+ def _desc(mem_eff_measured: bool, *, admit_peak: int | None = None) -> BackendDescriptor:
292
+ vram: dict = {"weights_gib": 1.7, "activation_gib": 3.9, "context_gib": 0.0}
293
+ if admit_peak:
294
+ vram["admit_peak_mib"] = admit_peak
295
+ return BackendDescriptor(
296
+ name="paint",
297
+ adapter="mock",
298
+ vram_mib=6000,
299
+ priority=40,
300
+ vram=vram,
301
+ peak_profile={"quant_mode": "sdnq-fp8", "load_kwargs": {"memory_efficient": mem_eff_measured}},
302
+ )
303
+
304
+ def test_measured_mem_eff_activation_not_discounted_twice(self) -> None:
305
+ mgr = self._mgr(self._desc(mem_eff_measured=True))
306
+ _w, activation = mgr.footprint_parts_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True)
307
+ assert activation == int(3.9 * 1024) # intacta — já foi medida em mem-eff
308
+ assert activation > int(3.9 * 1024 * 0.65) + 100 # sem duplo desconto
309
+
310
+ def test_measured_fp16_activation_gets_mem_eff_discount(self) -> None:
311
+ mgr = self._mgr(self._desc(mem_eff_measured=False))
312
+ _w, activation = mgr.footprint_parts_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True)
313
+ assert activation == max(512, int(int(3.9 * 1024) * 0.65))
314
+
315
+ def test_admit_peak_mib_wins_over_analytic_sum(self) -> None:
316
+ mgr = self._mgr(self._desc(mem_eff_measured=True, admit_peak=6144))
317
+ assert mgr.peak_vram_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True) == 6144
318
+
319
+ def test_without_admit_peak_uses_analytic_sum(self) -> None:
320
+ from vramd.vram_planner import peak_vram_mib as compute_peak_mib
321
+
322
+ mgr = self._mgr(self._desc(mem_eff_measured=True))
323
+ w, a = mgr.footprint_parts_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True)
324
+ assert mgr.peak_vram_mib("paint", quant_mode="sdnq-fp8", memory_efficient=True) == compute_peak_mib(w, a)
325
+
326
+
276
327
  class TestErrorRecovery:
277
328
  """Em erro de geração, o backend é descarregado para recovery."""
278
329
 
@@ -492,7 +492,92 @@ class TestBufferedStdoutLine:
492
492
  import time
493
493
 
494
494
  t0 = time.monotonic()
495
- with pytest.raises(SubprocessWorkerError, match="timeout idle"):
495
+ with pytest.raises(SubprocessWorkerError, match="worker wedged"):
496
496
  pool.generate("paint3d", {})
497
497
  elapsed = time.monotonic() - t0
498
498
  assert elapsed < 5.0, f"esperava post_done curto, demorou {elapsed:.1f}s"
499
+
500
+
501
+ class TestOomWatcher:
502
+ """Watcher de OOM-spin: conta só o que cresce no stderr durante o job."""
503
+
504
+ def test_scan_counts_only_new_lines(self, tmp_path):
505
+ from vramd.subprocess_pool import _WorkerState
506
+
507
+ pool = _make_pool(FakePopen(), log_path_fn=lambda b: tmp_path / "w.log")
508
+ log = tmp_path / "w.log"
509
+ log.write_text("expandable_segments: memory mapping failed with OOM\n" * 3, encoding="utf-8")
510
+ state = _WorkerState(backend="paint3d", log_path=log)
511
+
512
+ n, off = pool._scan_worker_oom(state, 0)
513
+ assert n == 3
514
+ assert off == log.stat().st_size
515
+
516
+ # Re-scan do mesmo offset: nada novo.
517
+ assert pool._scan_worker_oom(state, off) == (0, off)
518
+
519
+ with open(log, "a", encoding="utf-8") as fh:
520
+ fh.write("linha normal\nCUDA out of memory\n")
521
+ n2, off2 = pool._scan_worker_oom(state, off)
522
+ assert n2 == 1 # só o padrão novo conta
523
+ assert off2 == log.stat().st_size
524
+
525
+ def test_wedge_aborts_with_classified_error(self, tmp_path, monkeypatch):
526
+ """OOM-spin (8+ linhas novas, sem progresso) → abort em segundos."""
527
+ import threading
528
+
529
+ import vramd.subprocess_pool as sp
530
+
531
+ monkeypatch.setattr(sp, "_OOM_SPIN_QUIET_SEC", 2.0)
532
+ fake = FakePopen()
533
+ pool = _make_pool(
534
+ fake,
535
+ event_timeout_sec=60.0,
536
+ log_path_fn=lambda b: tmp_path / "worker.log",
537
+ )
538
+ fake.push_events('{"event": "ready", "vram_mib": 1300}')
539
+ pool.load("paint3d", "paint3d", {})
540
+ log = tmp_path / "worker.log"
541
+
542
+ def writer() -> None:
543
+ import time
544
+
545
+ time.sleep(1.0)
546
+ log.write_text("expandable_segments: memory mapping failed with OOM\n" * 9, encoding="utf-8")
547
+
548
+ t = threading.Thread(target=writer)
549
+ t.start()
550
+ try:
551
+ with pytest.raises(SubprocessWorkerError, match="worker wedged — OOM-spin"):
552
+ pool.generate("paint3d", {})
553
+ finally:
554
+ t.join()
555
+
556
+
557
+ class TestWorkerDeadPredicate:
558
+ """ "worker wedged" entra no predicado de requeue (worker morto transitório)."""
559
+
560
+ def test_wedge_messages_trigger_requeue(self) -> None:
561
+ from vramd.dispatcher import _is_worker_dead
562
+
563
+ assert _is_worker_dead({"error": "paint3d: worker wedged — OOM-spin (12 allocs, 40s sem progress)"})
564
+ assert _is_worker_dead({"error": "paint3d: worker wedged — idle 600s sem progress"})
565
+ assert not _is_worker_dead({"error": "erro genérico de geração"})
566
+
567
+
568
+ class TestWorkerRecycling:
569
+ """Worker persistente recicla após N jobs (fragmentação/estado não acumula)."""
570
+
571
+ def test_recycles_after_max_jobs(self):
572
+ fake = FakePopen()
573
+ pool = _make_pool(fake, worker_max_jobs=3)
574
+ fake.push_events('{"event": "ready", "vram_mib": 1300}')
575
+ pool.load("paint3d", "paint3d", {})
576
+ state = pool._workers["paint3d"]
577
+ for _i in range(3):
578
+ fake.push_events('{"event": "progress", "pct": 0.5}')
579
+ fake.push_events('{"event": "done", "result": {"status": "ok"}}')
580
+ out = pool.generate("paint3d", {})
581
+ assert out["status"] == "ok"
582
+ assert state.jobs_served == 3
583
+ assert state.proc is None # reciclado: próximo load re-spawna
@@ -0,0 +1,21 @@
1
+ """Consistência de versão: o literal __version__ acompanha o pyproject."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from pathlib import Path
7
+
8
+ import vramd
9
+
10
+
11
+ def _pyproject_version() -> str:
12
+ pyproject = Path(__file__).resolve().parents[1] / "pyproject.toml"
13
+ match = re.search(r'^version\s*=\s*"([^"]+)"', pyproject.read_text(encoding="utf-8"), re.MULTILINE)
14
+ assert match, "pyproject.toml sem version"
15
+ return match.group(1)
16
+
17
+
18
+ class TestVersion:
19
+ def test_dunder_version_matches_pyproject(self) -> None:
20
+ # 0.3.2 saiu com __version__="0.3.1": `vramd --version` mentia.
21
+ assert vramd.__version__ == _pyproject_version()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes