vramd 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vramd-0.3.0/src/vramd.egg-info → vramd-0.3.2}/PKG-INFO +10 -3
- {vramd-0.3.0 → vramd-0.3.2}/README.md +9 -2
- {vramd-0.3.0 → vramd-0.3.2}/pyproject.toml +1 -1
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/__init__.py +1 -1
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/backend_manager.py +32 -7
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/cli.py +25 -0
- vramd-0.3.2/src/vramd/data/backends.yaml +31 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/dispatcher.py +2 -1
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/server.py +6 -3
- {vramd-0.3.0 → vramd-0.3.2/src/vramd.egg-info}/PKG-INFO +10 -3
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd.egg-info/SOURCES.txt +1 -0
- vramd-0.3.2/tests/test_cli_start.py +48 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_peak_profile.py +53 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_registry.py +11 -5
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_registry_layers.py +22 -9
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_vram_admit.py +68 -0
- vramd-0.3.0/src/vramd/data/backends.yaml +0 -51
- {vramd-0.3.0 → vramd-0.3.2}/LICENSE +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/setup.cfg +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/__main__.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/adapter.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/calibrate/__init__.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/calibrate/analysis.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/calibrate/compare.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/calibrate/emit.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/calibrate/hw_auto.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/calibrate/runner.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/calibrate/sampler.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/calibrate/serde.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/client.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/data/test-image.png +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/data/test-mesh.glb +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/errors.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/footprints.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/gpu.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/hooks.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/idle_evictor.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/job_queue.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/learn.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/logging.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/mcp_server.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/process_guard.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/protocol.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/py.typed +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/registry.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/scheduler.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/stats.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/subprocess_pool.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/toolchain.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/top.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/vram_planner.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/worker/__init__.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/worker/adapter.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/worker/protocol.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd/worker/serve.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd.egg-info/dependency_links.txt +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd.egg-info/entry_points.txt +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd.egg-info/requires.txt +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/src/vramd.egg-info/top_level.txt +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_abort_progress.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_backend_manager.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_backend_manager_hybrid.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_calibrate_analysis.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_calibrate_cli.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_calibrate_emit_compare.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_calibrate_hw_auto.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_calibrate_runner.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_calibrate_sampler.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_calibrate_serde.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_cli_debug_stats_bench.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_gpu_ids_load.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_hooks.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_idle_evictor.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_job_queue.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_job_wal.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_learn.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_learn_integration.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_mcp_server.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_phase_progress.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_process_guard.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_protocol_env.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_queue_metrics_eta.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_respawn.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_runtime_spawn.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_scheduler.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_server.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_stats.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_subprocess_integration.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_subprocess_per_backend_e2e.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_subprocess_pool.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_toolchain.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_top.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_unload_frees_vram.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_vram_planner.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_vramd_coverage_100.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_worker.py +0 -0
- {vramd-0.3.0 → vramd-0.3.2}/tests/test_zero_vram.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vramd
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: VRAM admission control for generative inference on consumer GPUs
|
|
5
5
|
Author: Maikeu Locatelli
|
|
6
6
|
License-Expression: MIT
|
|
@@ -51,7 +51,8 @@ optimizes *fitting*.
|
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
53
|
pip install vramd # 9 MB — the supervisor doesn't import torch
|
|
54
|
-
|
|
54
|
+
export VRAMD_BACKENDS_FILE=/path/to/backends.yaml # your registry
|
|
55
|
+
vramd start & # refuses to start with zero backends configured
|
|
55
56
|
vramd submit my-model --prompt "…" --wait
|
|
56
57
|
```
|
|
57
58
|
|
|
@@ -271,9 +272,15 @@ kills the wrong workload.
|
|
|
271
272
|
## Configuration
|
|
272
273
|
|
|
273
274
|
```
|
|
274
|
-
data/backends.yaml (
|
|
275
|
+
data/backends.yaml (empty) → $VRAMD_BACKENDS_FILE → ~/.config/vramd/backends.d/*.yaml
|
|
275
276
|
```
|
|
276
277
|
|
|
278
|
+
The packaged registry is **empty on purpose**: `vramd start` refuses to serve
|
|
279
|
+
fake example backends (they'd grab the socket and real delegation would fail
|
|
280
|
+
with opaque errors). Descriptor examples live in
|
|
281
|
+
[`examples/backends.yaml`](examples/backends.yaml); `vramd start --allow-empty`
|
|
282
|
+
escapes the guard for tests.
|
|
283
|
+
|
|
277
284
|
Per-key overlay: a file with `{name: x, vram_mib: 5632}` fixes only that field
|
|
278
285
|
and inherits the rest. That's how a calibrated descriptor takes effect without
|
|
279
286
|
editing the package.
|
|
@@ -18,7 +18,8 @@ optimizes *fitting*.
|
|
|
18
18
|
|
|
19
19
|
```bash
|
|
20
20
|
pip install vramd # 9 MB — the supervisor doesn't import torch
|
|
21
|
-
|
|
21
|
+
export VRAMD_BACKENDS_FILE=/path/to/backends.yaml # your registry
|
|
22
|
+
vramd start & # refuses to start with zero backends configured
|
|
22
23
|
vramd submit my-model --prompt "…" --wait
|
|
23
24
|
```
|
|
24
25
|
|
|
@@ -238,9 +239,15 @@ kills the wrong workload.
|
|
|
238
239
|
## Configuration
|
|
239
240
|
|
|
240
241
|
```
|
|
241
|
-
data/backends.yaml (
|
|
242
|
+
data/backends.yaml (empty) → $VRAMD_BACKENDS_FILE → ~/.config/vramd/backends.d/*.yaml
|
|
242
243
|
```
|
|
243
244
|
|
|
245
|
+
The packaged registry is **empty on purpose**: `vramd start` refuses to serve
|
|
246
|
+
fake example backends (they'd grab the socket and real delegation would fail
|
|
247
|
+
with opaque errors). Descriptor examples live in
|
|
248
|
+
[`examples/backends.yaml`](examples/backends.yaml); `vramd start --allow-empty`
|
|
249
|
+
escapes the guard for tests.
|
|
250
|
+
|
|
244
251
|
Per-key overlay: a file with `{name: x, vram_mib: 5632}` fixes only that field
|
|
245
252
|
and inherits the rest. That's how a calibrated descriptor takes effect without
|
|
246
253
|
editing the package.
|
|
@@ -666,12 +666,22 @@ class BackendManager:
|
|
|
666
666
|
quant_mode: str = "none",
|
|
667
667
|
memory_efficient: bool = False,
|
|
668
668
|
group_offload: bool = False,
|
|
669
|
+
streams_on_load: bool = False,
|
|
669
670
|
footprint_key: str | None = None,
|
|
670
671
|
) -> tuple[int, int]:
|
|
671
672
|
"""(weights_mib, activation_mib) a partir do footprint ou YAML.
|
|
672
673
|
|
|
673
674
|
``footprint_key``: override por request (ex.: text2d 4B vs 9B — a chave
|
|
674
675
|
do descriptor é estática e não sabe qual modelo o hw_auto escolheu).
|
|
676
|
+
|
|
677
|
+
``group_offload`` (text3d SDNQ) reduz o residente mas a activação
|
|
678
|
+
streama em chunks dinâmicos que crescem na VRAM livre pós-offload →
|
|
679
|
+
activação completa, SEM desconto mem-eff. ``streams_on_load``
|
|
680
|
+
(diffusers model_cpu offload, text2d) é o próprio caminho
|
|
681
|
+
memory-efficient → o desconto 0.65 aplica na mesma. Regressão 0.3.0:
|
|
682
|
+
os callers dobravam ``streams`` em ``group_offload`` e o desconto
|
|
683
|
+
desaparecia — o admit pedia 6117 MiB (activação medida intacta) e
|
|
684
|
+
recusava text2d numa 6 GB que a própria calibração tinha medido.
|
|
675
685
|
"""
|
|
676
686
|
desc = self._registry.descriptor(name)
|
|
677
687
|
|
|
@@ -694,11 +704,16 @@ class BackendManager:
|
|
|
694
704
|
from vramd.footprints import get_footprint
|
|
695
705
|
|
|
696
706
|
fp = get_footprint(fp_key)
|
|
697
|
-
if group_offload:
|
|
698
|
-
# group+stream: pico ≈ maior
|
|
699
|
-
#
|
|
707
|
+
if group_offload or streams_on_load:
|
|
708
|
+
# group+stream / load streaming: pico de warmup ≈ maior
|
|
709
|
+
# leaf/módulo onloaded + activação. No group_offload os
|
|
710
|
+
# chunks dinâmicos usam a VRAM livre pós-offload → activação
|
|
711
|
+
# completa; no load streaming (diffusers offload) o caminho
|
|
712
|
+
# É memory-efficient → desconto aplica.
|
|
700
713
|
weights = int(fp.largest_gib(quant_mode) * 1024)
|
|
701
714
|
activation = int(fp.activation_gib * 1024)
|
|
715
|
+
if streams_on_load and memory_efficient and not group_offload:
|
|
716
|
+
activation = max(512, int(activation * _MEMORY_EFFICIENT_ACTIVATION_FACTOR))
|
|
702
717
|
return max(256, weights), max(512, activation)
|
|
703
718
|
weights = int(fp.weights_gib(quant_mode) * 1024)
|
|
704
719
|
activation = int(fp.activation_gib * 1024)
|
|
@@ -717,9 +732,11 @@ class BackendManager:
|
|
|
717
732
|
weights = int(weights * QUANT_WEIGHT_FACTOR.get(quant_mode, 1.0))
|
|
718
733
|
except Exception:
|
|
719
734
|
pass
|
|
720
|
-
if group_offload:
|
|
735
|
+
if group_offload or streams_on_load:
|
|
721
736
|
# Sem footprint: aproximar onloaded ≈ 40% dos pesos (como largest default).
|
|
722
737
|
weights = max(256, int(weights * 0.4))
|
|
738
|
+
if streams_on_load and memory_efficient and not group_offload:
|
|
739
|
+
activation = max(512, int(activation * _MEMORY_EFFICIENT_ACTIVATION_FACTOR))
|
|
723
740
|
return weights, max(512, activation)
|
|
724
741
|
if memory_efficient and not group_offload:
|
|
725
742
|
activation = max(512, int(activation * _MEMORY_EFFICIENT_ACTIVATION_FACTOR))
|
|
@@ -732,6 +749,7 @@ class BackendManager:
|
|
|
732
749
|
quant_mode: str = "none",
|
|
733
750
|
memory_efficient: bool = False,
|
|
734
751
|
group_offload: bool = False,
|
|
752
|
+
streams_on_load: bool = False,
|
|
735
753
|
footprint_key: str | None = None,
|
|
736
754
|
) -> int:
|
|
737
755
|
"""Pico = pesos(quant) + activação de inferência + safety."""
|
|
@@ -740,6 +758,7 @@ class BackendManager:
|
|
|
740
758
|
quant_mode=quant_mode,
|
|
741
759
|
memory_efficient=memory_efficient,
|
|
742
760
|
group_offload=group_offload,
|
|
761
|
+
streams_on_load=streams_on_load,
|
|
743
762
|
footprint_key=footprint_key,
|
|
744
763
|
)
|
|
745
764
|
return compute_peak_mib(weights, activation)
|
|
@@ -819,11 +838,15 @@ class BackendManager:
|
|
|
819
838
|
# largest-module + activação. ``group_offload`` só reduz o residente
|
|
820
839
|
# *depois* do load (headroom) — usar largest no admit de um backend que
|
|
821
840
|
# carrega tudo subestima o warmup → OOM a meio do adapter.load.
|
|
841
|
+
# ``streams`` viaja SEPARADO de ``group_offload``: o load streaming é o
|
|
842
|
+
# caminho memory-efficient (desconto 0.65 na activação), o group
|
|
843
|
+
# offload real exige activação completa (chunks dinâmicos).
|
|
822
844
|
weights_mib, activation_mib = self.footprint_parts_mib(
|
|
823
845
|
name,
|
|
824
846
|
quant_mode=quant,
|
|
825
847
|
memory_efficient=mem_eff,
|
|
826
|
-
group_offload=
|
|
848
|
+
group_offload=group_off,
|
|
849
|
+
streams_on_load=streams,
|
|
827
850
|
footprint_key=load_kwargs.get("footprint_key"),
|
|
828
851
|
)
|
|
829
852
|
peak = compute_peak_mib(weights_mib, activation_mib)
|
|
@@ -1087,7 +1110,8 @@ class BackendManager:
|
|
|
1087
1110
|
name,
|
|
1088
1111
|
quant_mode=_quant,
|
|
1089
1112
|
memory_efficient=mem_eff,
|
|
1090
|
-
group_offload=group_off
|
|
1113
|
+
group_offload=group_off,
|
|
1114
|
+
streams_on_load=streams,
|
|
1091
1115
|
footprint_key=req.get("footprint_key"),
|
|
1092
1116
|
)
|
|
1093
1117
|
err = InsufficientVramError(
|
|
@@ -1578,7 +1602,8 @@ class BackendManager:
|
|
|
1578
1602
|
backend,
|
|
1579
1603
|
quant_mode=quant,
|
|
1580
1604
|
memory_efficient=mem_eff,
|
|
1581
|
-
group_offload=
|
|
1605
|
+
group_offload=_go,
|
|
1606
|
+
streams_on_load=streams,
|
|
1582
1607
|
footprint_key=src.get("footprint_key"),
|
|
1583
1608
|
),
|
|
1584
1609
|
)
|
|
@@ -162,12 +162,18 @@ def cli() -> None:
|
|
|
162
162
|
help="Segundos sem uso antes de terminar o subprocesso worker (0 desliga).",
|
|
163
163
|
)
|
|
164
164
|
@click.option("--verbose", "-v", is_flag=True, help="Logs detalhados")
|
|
165
|
+
@click.option(
|
|
166
|
+
"--allow-empty",
|
|
167
|
+
is_flag=True,
|
|
168
|
+
help="Arranca mesmo com zero backends (tests/dev — cada delegação falhará).",
|
|
169
|
+
)
|
|
165
170
|
def start_cmd(
|
|
166
171
|
socket_path: str | None,
|
|
167
172
|
idle_timeout_min: int,
|
|
168
173
|
idle_evict_sec: float,
|
|
169
174
|
worker_shutdown_sec: float,
|
|
170
175
|
verbose: bool,
|
|
176
|
+
allow_empty: bool,
|
|
171
177
|
) -> None:
|
|
172
178
|
"""Arranca o vramd (foreground)."""
|
|
173
179
|
from vramd.logging import configure_logging
|
|
@@ -181,6 +187,25 @@ def start_cmd(
|
|
|
181
187
|
sys.exit(1)
|
|
182
188
|
|
|
183
189
|
registry = Registry()
|
|
190
|
+
if not registry.names and not allow_empty:
|
|
191
|
+
# Um vramd sem backends é uma armadilha: agarra o socket, o auto-start
|
|
192
|
+
# das tools passa a aceitá-lo como "ativo" e TODA a delegação falha
|
|
193
|
+
# com "backend desconhecido". Recusar aqui é o diagnóstico.
|
|
194
|
+
console.print(
|
|
195
|
+
Panel.fit(
|
|
196
|
+
"[bold red]✗ Nenhum backend configurado — o vramd não arranca.[/bold red]\n\n"
|
|
197
|
+
"O registry empacotado é vazio de propósito (nunca serve exemplos "
|
|
198
|
+
"fictícios). Aponta o vramd para o teu registry:\n\n"
|
|
199
|
+
" • [cyan]VRAMD_BACKENDS_FILE[/cyan]=ficheiro.yaml (um ou mais, separados por ':')\n"
|
|
200
|
+
" • overlays em [cyan]~/.config/vramd/backends.d/*.yaml[/cyan]\n\n"
|
|
201
|
+
"Um descriptor calibrado sai de [cyan]vramd calibrate <backend> --out[/cyan].\n"
|
|
202
|
+
"No monorepo AiGameKit as tools auto-arrancam o vramd já configurado — "
|
|
203
|
+
"basta [cyan]vramd stop[/cyan] se uma instância vazia estiver agarrada ao socket.\n\n"
|
|
204
|
+
"[dim]Para arrancar na mesma (tests): vramd start --allow-empty[/dim]",
|
|
205
|
+
border_style="red",
|
|
206
|
+
)
|
|
207
|
+
)
|
|
208
|
+
sys.exit(1)
|
|
184
209
|
log_line = f"Log: [cyan]{log_path}[/cyan]\n" if log_path else ""
|
|
185
210
|
console.print(
|
|
186
211
|
Panel.fit(
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Registry de backends do vramd — VAZIO de propósito.
|
|
2
|
+
#
|
|
3
|
+
# O pacote não traz backends de ninguém. Um `vramd start` sem configuração
|
|
4
|
+
# recusa arrancar ("nenhum backend") em vez de servir exemplos fictícios:
|
|
5
|
+
# uma instância dessas agarra o socket e a delegação das tools reais falha
|
|
6
|
+
# com erros opacos (backend desconhecido).
|
|
7
|
+
#
|
|
8
|
+
# O teu registry entra por sobreposição:
|
|
9
|
+
#
|
|
10
|
+
# $VRAMD_BACKENDS_FILE um ou mais ficheiros (separados por ':')
|
|
11
|
+
# ~/.config/vramd/backends.d/*.yaml uma pasta (ordem alfabética)
|
|
12
|
+
#
|
|
13
|
+
# A sobreposição é **por chave**: um ficheiro com `{name: x, vram_mib: 5632}`
|
|
14
|
+
# corrige só esse campo e herda o resto. É assim que um descriptor calibrado
|
|
15
|
+
# (`vramd calibrate --out`) entra em vigor sem editar nada aqui.
|
|
16
|
+
#
|
|
17
|
+
# Exemplos completos de descriptors: examples/backends.yaml no repositório.
|
|
18
|
+
#
|
|
19
|
+
# Campos:
|
|
20
|
+
# name identificador do backend
|
|
21
|
+
# adapter dotted path do módulo com a classe `Adapter` (só modo in-process)
|
|
22
|
+
# vram_mib pico estimado; substituído pelo bloco `vram:` quando medido
|
|
23
|
+
# priority evicção: maior = manter carregado; menor = evictado primeiro
|
|
24
|
+
# runtime como arrancar o worker (command/cwd/env/timeouts)
|
|
25
|
+
# load_keys kwargs do request que influenciam a carga deste backend
|
|
26
|
+
# shape_keys subconjunto que força reload quando muda
|
|
27
|
+
# vram MEDIDO por `vramd calibrate` — vence a estimativa no admit
|
|
28
|
+
# peak_profile quant sob a qual foi medido + sinais (staged, unload_frees_vram)
|
|
29
|
+
|
|
30
|
+
version: 2
|
|
31
|
+
backends: []
|
|
@@ -614,7 +614,8 @@ class VramdServer:
|
|
|
614
614
|
name,
|
|
615
615
|
quant_mode=quant,
|
|
616
616
|
memory_efficient=mem_eff,
|
|
617
|
-
group_offload=group_off
|
|
617
|
+
group_offload=group_off,
|
|
618
|
+
streams_on_load=streams,
|
|
618
619
|
footprint_key=load_kwargs.get("footprint_key"),
|
|
619
620
|
),
|
|
620
621
|
"quant_mode": quant,
|
|
@@ -789,7 +790,8 @@ class VramdServer:
|
|
|
789
790
|
bname,
|
|
790
791
|
quant_mode=quant,
|
|
791
792
|
memory_efficient=mem_eff,
|
|
792
|
-
group_offload=
|
|
793
|
+
group_offload=group_off,
|
|
794
|
+
streams_on_load=streams,
|
|
793
795
|
footprint_key=request.get("footprint_key"),
|
|
794
796
|
),
|
|
795
797
|
)
|
|
@@ -819,7 +821,8 @@ class VramdServer:
|
|
|
819
821
|
str(backend),
|
|
820
822
|
quant_mode=quant,
|
|
821
823
|
memory_efficient=mem_eff,
|
|
822
|
-
group_offload=group_off
|
|
824
|
+
group_offload=group_off,
|
|
825
|
+
streams_on_load=streams,
|
|
823
826
|
footprint_key=request.get("footprint_key"),
|
|
824
827
|
)
|
|
825
828
|
if backend and self.registry.has(str(backend))
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vramd
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: VRAM admission control for generative inference on consumer GPUs
|
|
5
5
|
Author: Maikeu Locatelli
|
|
6
6
|
License-Expression: MIT
|
|
@@ -51,7 +51,8 @@ optimizes *fitting*.
|
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
53
|
pip install vramd # 9 MB — the supervisor doesn't import torch
|
|
54
|
-
|
|
54
|
+
export VRAMD_BACKENDS_FILE=/path/to/backends.yaml # your registry
|
|
55
|
+
vramd start & # refuses to start with zero backends configured
|
|
55
56
|
vramd submit my-model --prompt "…" --wait
|
|
56
57
|
```
|
|
57
58
|
|
|
@@ -271,9 +272,15 @@ kills the wrong workload.
|
|
|
271
272
|
## Configuration
|
|
272
273
|
|
|
273
274
|
```
|
|
274
|
-
data/backends.yaml (
|
|
275
|
+
data/backends.yaml (empty) → $VRAMD_BACKENDS_FILE → ~/.config/vramd/backends.d/*.yaml
|
|
275
276
|
```
|
|
276
277
|
|
|
278
|
+
The packaged registry is **empty on purpose**: `vramd start` refuses to serve
|
|
279
|
+
fake example backends (they'd grab the socket and real delegation would fail
|
|
280
|
+
with opaque errors). Descriptor examples live in
|
|
281
|
+
[`examples/backends.yaml`](examples/backends.yaml); `vramd start --allow-empty`
|
|
282
|
+
escapes the guard for tests.
|
|
283
|
+
|
|
277
284
|
Per-key overlay: a file with `{name: x, vram_mib: 5632}` fixes only that field
|
|
278
285
|
and inherits the rest. That's how a calibrated descriptor takes effect without
|
|
279
286
|
editing the package.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Guard de arranque: um registry vazio recusa servir (--allow-empty para tests)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from unittest.mock import MagicMock, patch
|
|
7
|
+
|
|
8
|
+
from click.testing import CliRunner
|
|
9
|
+
|
|
10
|
+
from vramd.cli import cli
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestStartRefusesEmptyRegistry:
|
|
14
|
+
def _bare_env(self, monkeypatch, tmp_path: Path) -> None:
|
|
15
|
+
"""Sem registry: package vazio, sem overlays do utilizador."""
|
|
16
|
+
monkeypatch.delenv("VRAMD_BACKENDS_FILE", raising=False)
|
|
17
|
+
monkeypatch.setenv("VRAMD_BACKENDS_DIR", str(tmp_path / "vazio"))
|
|
18
|
+
|
|
19
|
+
def test_empty_registry_exits_nonzero_with_hint(self, tmp_path: Path, monkeypatch) -> None:
|
|
20
|
+
self._bare_env(monkeypatch, tmp_path)
|
|
21
|
+
result = CliRunner().invoke(cli, ["start", "--socket", str(tmp_path / "t.sock")])
|
|
22
|
+
assert result.exit_code == 1
|
|
23
|
+
assert "Nenhum backend configurado" in result.output
|
|
24
|
+
# O diagnóstico diz COMO configurar (senão o operador fica na mesma).
|
|
25
|
+
assert "VRAMD_BACKENDS_FILE" in result.output
|
|
26
|
+
assert "--allow-empty" in result.output
|
|
27
|
+
|
|
28
|
+
def test_allow_empty_proceeds_to_server(self, tmp_path: Path, monkeypatch) -> None:
|
|
29
|
+
self._bare_env(monkeypatch, tmp_path)
|
|
30
|
+
server = MagicMock()
|
|
31
|
+
with patch("vramd.server.VramdServer", return_value=server):
|
|
32
|
+
result = CliRunner().invoke(cli, ["start", "--socket", str(tmp_path / "t.sock"), "--allow-empty"])
|
|
33
|
+
assert result.exit_code == 0
|
|
34
|
+
server.serve_forever.assert_called_once()
|
|
35
|
+
|
|
36
|
+
def test_registry_with_backends_starts_without_flag(self, tmp_path: Path, monkeypatch) -> None:
|
|
37
|
+
(tmp_path / "base.yaml").write_text(
|
|
38
|
+
"version: 2\nbackends:\n - name: demo\n adapter: pkg.demo\n vram_mib: 1024\n priority: 10\n",
|
|
39
|
+
encoding="utf-8",
|
|
40
|
+
)
|
|
41
|
+
monkeypatch.setenv("VRAMD_BACKENDS_FILE", str(tmp_path / "base.yaml"))
|
|
42
|
+
monkeypatch.setenv("VRAMD_BACKENDS_DIR", str(tmp_path / "vazio"))
|
|
43
|
+
server = MagicMock()
|
|
44
|
+
with patch("vramd.server.VramdServer", return_value=server):
|
|
45
|
+
result = CliRunner().invoke(cli, ["start", "--socket", str(tmp_path / "t.sock")])
|
|
46
|
+
assert result.exit_code == 0
|
|
47
|
+
assert "demo" in result.output
|
|
48
|
+
server.serve_forever.assert_called_once()
|
|
@@ -167,3 +167,56 @@ class TestMeasuredFootprintWins:
|
|
|
167
167
|
manager = manager_with(footprint_key="stable-audio-open")
|
|
168
168
|
weights, activation = manager.footprint_parts_mib("d")
|
|
169
169
|
assert weights > 0 and activation > 0
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
class TestStreamsOnLoadKeepsMemEffDiscount:
|
|
173
|
+
"""Regressão 0.3.0: ``streams_on_load`` chegava dobrado como ``group_offload``
|
|
174
|
+
e travava o desconto 0.65 da activação — o admit pedia 6117 MiB
|
|
175
|
+
(429+5304+safety) e recusava TODO o text2d numa GPU de 6 GB (livre ~5736
|
|
176
|
+
com desktop) que a própria calibração (`backends-6g.yaml`) tinha corrido."""
|
|
177
|
+
|
|
178
|
+
def _calibrated_text2d(self) -> BackendManager:
|
|
179
|
+
# Números reais do catálogo backends-6g.yaml (RTX 4050, 2026-08-08):
|
|
180
|
+
# pesos 0.19 + contexto 0.23 + activação 5.18 (staged load, inflada
|
|
181
|
+
# pela fragmentação do allocator — medição de confiança baixa).
|
|
182
|
+
return manager_with(
|
|
183
|
+
vram={"weights_gib": 0.19, "activation_gib": 5.18, "context_gib": 0.23},
|
|
184
|
+
peak_profile={"quant_mode": "sdnq-int4"},
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
def test_measured_activation_discounted_with_streams_and_mem_eff(self):
|
|
188
|
+
from vramd.backend_manager import _MEMORY_EFFICIENT_ACTIVATION_FACTOR
|
|
189
|
+
|
|
190
|
+
manager = self._calibrated_text2d()
|
|
191
|
+
weights, activation = manager.footprint_parts_mib(
|
|
192
|
+
"d", quant_mode="sdnq-int4", memory_efficient=True, streams_on_load=True
|
|
193
|
+
)
|
|
194
|
+
assert weights == int(0.19 * 1024) + int(0.23 * 1024)
|
|
195
|
+
assert activation == max(512, int(5.18 * 1024 * _MEMORY_EFFICIENT_ACTIVATION_FACTOR))
|
|
196
|
+
|
|
197
|
+
def test_calibrated_text2d_peak_fits_a_6gb_gpu(self):
|
|
198
|
+
manager = self._calibrated_text2d()
|
|
199
|
+
peak = manager.peak_vram_mib("d", quant_mode="sdnq-int4", memory_efficient=True, streams_on_load=True)
|
|
200
|
+
# 6 GB com sessão de desktop (~400 MiB) deixa ~5736 MiB livres.
|
|
201
|
+
assert peak <= 5736
|
|
202
|
+
|
|
203
|
+
def test_group_offload_still_demands_full_activation(self):
|
|
204
|
+
"""Chunks dinâmicos do group offload real crescem na VRAM livre → sem desconto."""
|
|
205
|
+
manager = self._calibrated_text2d()
|
|
206
|
+
_, activation = manager.footprint_parts_mib(
|
|
207
|
+
"d", quant_mode="sdnq-int4", memory_efficient=True, group_offload=True
|
|
208
|
+
)
|
|
209
|
+
assert activation == int(5.18 * 1024)
|
|
210
|
+
|
|
211
|
+
def test_estimated_streams_uses_largest_module_and_discount(self):
|
|
212
|
+
from vramd.footprints import get_footprint
|
|
213
|
+
|
|
214
|
+
manager = manager_with(footprint_key="flux-klein-4b")
|
|
215
|
+
fp = get_footprint("flux-klein-4b")
|
|
216
|
+
weights, activation = manager.footprint_parts_mib(
|
|
217
|
+
"d", quant_mode="sdnq-int4", memory_efficient=True, streams_on_load=True
|
|
218
|
+
)
|
|
219
|
+
# Warmup de load streaming (diffusers offload) = maior módulo + activação
|
|
220
|
+
# com desconto mem-eff — não pesos completos nem activação completa.
|
|
221
|
+
assert weights == max(256, int(fp.largest_gib("sdnq-int4") * 1024))
|
|
222
|
+
assert activation == max(512, int(fp.activation_gib * 1024 * 0.65))
|
|
@@ -13,18 +13,24 @@ from vramd.registry import BackendDescriptor, Registry, load_descriptors
|
|
|
13
13
|
class TestLoadDescriptors:
|
|
14
14
|
"""Carregar descriptors do backends.yaml."""
|
|
15
15
|
|
|
16
|
-
def
|
|
17
|
-
"""O YAML empacotado é
|
|
16
|
+
def test_packaged_registry_is_empty(self) -> None:
|
|
17
|
+
"""O YAML empacotado é VAZIO — exemplos fictícios nunca são servidos.
|
|
18
|
+
|
|
19
|
+
Um vramd arrancado sem configuração recusa arrancar; servir exemplos
|
|
20
|
+
agarrava o socket e fazia a delegação das tools reais falhar com
|
|
21
|
+
"backend desconhecido".
|
|
22
|
+
"""
|
|
18
23
|
from vramd.registry import _default_yaml_path
|
|
19
24
|
|
|
20
25
|
descs = load_descriptors(_default_yaml_path())
|
|
21
|
-
assert
|
|
26
|
+
assert descs == {}
|
|
22
27
|
|
|
23
|
-
def
|
|
28
|
+
def test_user_registry_is_sole_source(self) -> None:
|
|
24
29
|
"""O fixture dos testes entra por VRAMD_BACKENDS_FILE, como o de um utilizador."""
|
|
25
30
|
descs = load_descriptors()
|
|
26
31
|
assert {"text3d", "paint3d", "motion3d"} <= set(descs)
|
|
27
|
-
|
|
32
|
+
# O package vazio não injeta exemplos por baixo do registry do utilizador.
|
|
33
|
+
assert not any(name.startswith("example-") for name in descs)
|
|
28
34
|
|
|
29
35
|
def test_descriptors_have_required_fields(self) -> None:
|
|
30
36
|
descs = load_descriptors()
|
|
@@ -111,17 +111,21 @@ class TestLoadDescriptors:
|
|
|
111
111
|
base = write_yaml(tmp_path / "base.yaml", [BASE_ENTRY])
|
|
112
112
|
override = write_yaml(tmp_path / "over.yaml", [{"name": "demo", "vram_mib": 5632}])
|
|
113
113
|
monkeypatch.setenv(ENV_BACKENDS_FILE, os.pathsep.join([base, override]))
|
|
114
|
-
#
|
|
114
|
+
# O package (vazio) entra também por baixo; o que conta é base→override.
|
|
115
115
|
descs = load_descriptors()
|
|
116
116
|
assert descs["demo"].vram_mib == 5632
|
|
117
117
|
assert descs["demo"].priority == 10
|
|
118
118
|
|
|
119
|
-
def
|
|
120
|
-
|
|
121
|
-
|
|
119
|
+
def test_base_backend_can_be_recalibrated_by_a_later_file(self, tmp_path, monkeypatch, isolated_env):
|
|
120
|
+
base = write_yaml(
|
|
121
|
+
tmp_path / "base.yaml",
|
|
122
|
+
[{"name": "demo", "adapter": "pkg.demo", "vram_mib": 6000, "priority": 40, "tool": "my_diffusion"}],
|
|
123
|
+
)
|
|
124
|
+
override = write_yaml(tmp_path / "cal.yaml", [{"name": "demo", "vram_mib": 5632}])
|
|
125
|
+
monkeypatch.setenv(ENV_BACKENDS_FILE, os.pathsep.join([base, override]))
|
|
122
126
|
descs = load_descriptors()
|
|
123
|
-
assert descs["
|
|
124
|
-
assert descs["
|
|
127
|
+
assert descs["demo"].vram_mib == 5632
|
|
128
|
+
assert descs["demo"].tool == "my_diffusion" # herdado da camada de baixo
|
|
125
129
|
|
|
126
130
|
def test_missing_required_field_after_merge_raises(self, tmp_path, isolated_env):
|
|
127
131
|
path = write_yaml(tmp_path / "mau.yaml", [{"name": "x", "vram_mib": 1}])
|
|
@@ -246,9 +250,18 @@ class TestDescriptorHelpers:
|
|
|
246
250
|
|
|
247
251
|
class TestRegistryWithLayers:
|
|
248
252
|
def test_registry_uses_merged_descriptors(self, tmp_path, monkeypatch, isolated_env):
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
253
|
+
base = write_yaml(
|
|
254
|
+
tmp_path / "base.yaml",
|
|
255
|
+
[{"name": "demo", "adapter": "pkg.demo", "vram_mib": 6000, "priority": 40}],
|
|
256
|
+
)
|
|
257
|
+
override = write_yaml(tmp_path / "cal.yaml", [{"name": "demo", "vram_mib": 5760}])
|
|
258
|
+
monkeypatch.setenv(ENV_BACKENDS_FILE, os.pathsep.join([base, override]))
|
|
259
|
+
assert Registry().descriptor("demo").vram_mib == 5760
|
|
260
|
+
|
|
261
|
+
def test_registry_with_no_sources_is_empty(self, monkeypatch, isolated_env):
|
|
262
|
+
monkeypatch.delenv(ENV_BACKENDS_FILE, raising=False)
|
|
263
|
+
monkeypatch.setenv(ENV_BACKENDS_DIR, str(isolated_env / "vazio"))
|
|
264
|
+
assert Registry().names == []
|
|
252
265
|
|
|
253
266
|
def test_external_backend_registers_without_touching_the_package(self, tmp_path, monkeypatch, isolated_env):
|
|
254
267
|
entry = {
|
|
@@ -123,3 +123,71 @@ class TestPaint3dMemoryEfficientAdmit:
|
|
|
123
123
|
assert loaded["kwargs"].get("memory_efficient") is True
|
|
124
124
|
peak = mgr.peak_vram_mib("paint3d", quant_mode="sdnq-uint8", memory_efficient=True)
|
|
125
125
|
assert peak <= 5657
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class TestText2dStreamsOnLoadAdmit:
|
|
129
|
+
"""Regressão 0.3.0 (RTX 4050 6 GB, catálogo backends-6g.yaml): text2d
|
|
130
|
+
calibrado em sdnq-int4 com load streaming (diffusers offload) era recusado
|
|
131
|
+
com peak=6117 MiB contra ~5736 MiB livres — ``streams_on_load`` dobrado
|
|
132
|
+
como ``group_offload`` desligava o desconto 0.65 da activação medida."""
|
|
133
|
+
|
|
134
|
+
@staticmethod
|
|
135
|
+
def _registry() -> Registry:
|
|
136
|
+
from vramd.registry import BackendDescriptor
|
|
137
|
+
|
|
138
|
+
return Registry(
|
|
139
|
+
descriptors={
|
|
140
|
+
"text2d": BackendDescriptor(
|
|
141
|
+
name="text2d",
|
|
142
|
+
adapter="a",
|
|
143
|
+
vram_mib=5760,
|
|
144
|
+
priority=25,
|
|
145
|
+
vram={"weights_gib": 0.19, "activation_gib": 5.18, "context_gib": 0.23},
|
|
146
|
+
peak_profile={
|
|
147
|
+
"quant_mode": "sdnq-int4",
|
|
148
|
+
"memory_efficient_with_quant": True,
|
|
149
|
+
"streams_on_load_with_memory_efficient": True,
|
|
150
|
+
},
|
|
151
|
+
)
|
|
152
|
+
}
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
@pytest.fixture(autouse=True)
|
|
156
|
+
def _no_admit_wait(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
157
|
+
monkeypatch.setattr(P, "VRAM_ADMIT_WAIT_SEC", 0.0)
|
|
158
|
+
|
|
159
|
+
def test_calibrated_text2d_admits_on_6gb_free(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
160
|
+
monkeypatch.setenv("VRAMD_VRAM_SAFETY_MIB", "384")
|
|
161
|
+
loaded: dict[str, object] = {}
|
|
162
|
+
|
|
163
|
+
class _FakeAdapter:
|
|
164
|
+
def load(self, **kwargs):
|
|
165
|
+
loaded["ok"] = True
|
|
166
|
+
return object()
|
|
167
|
+
|
|
168
|
+
def unload(self, model):
|
|
169
|
+
pass
|
|
170
|
+
|
|
171
|
+
mgr = BackendManager(self._registry(), query_free_mib=lambda: 5736, clear_vram=lambda: None)
|
|
172
|
+
monkeypatch.setattr(mgr._registry, "adapter", lambda name: _FakeAdapter())
|
|
173
|
+
model = mgr.ensure_loaded("text2d", sdnq_preset="sdnq-int4", memory_efficient=True)
|
|
174
|
+
assert model is not None
|
|
175
|
+
assert loaded.get("ok") is True
|
|
176
|
+
|
|
177
|
+
def test_still_refuses_when_free_below_discounted_peak(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
178
|
+
monkeypatch.setenv("VRAMD_VRAM_SAFETY_MIB", "384")
|
|
179
|
+
|
|
180
|
+
class _FakeAdapter:
|
|
181
|
+
def load(self, **kwargs):
|
|
182
|
+
return object()
|
|
183
|
+
|
|
184
|
+
def unload(self, model):
|
|
185
|
+
pass
|
|
186
|
+
|
|
187
|
+
mgr = BackendManager(self._registry(), query_free_mib=lambda: 4000, clear_vram=lambda: None)
|
|
188
|
+
monkeypatch.setattr(mgr._registry, "adapter", lambda name: _FakeAdapter())
|
|
189
|
+
with pytest.raises(InsufficientVramError) as ei:
|
|
190
|
+
mgr.ensure_loaded("text2d", sdnq_preset="sdnq-int4", memory_efficient=True)
|
|
191
|
+
# Descontado (429+3447+384=4260) mas ainda acima de 4000 → o admit guarda.
|
|
192
|
+
assert ei.value.peak_mib == 4260
|
|
193
|
+
assert ei.value.activation_mib == 3447
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
# Registry de backends do vramd — este ficheiro é só um EXEMPLO.
|
|
2
|
-
#
|
|
3
|
-
# O teu registry vive fora do pacote, por sobreposição:
|
|
4
|
-
#
|
|
5
|
-
# $VRAMD_BACKENDS_FILE um ficheiro
|
|
6
|
-
# ~/.config/vramd/backends.d/*.yaml uma pasta (ordem alfabética)
|
|
7
|
-
#
|
|
8
|
-
# A sobreposição é **por chave**: um ficheiro com `{name: x, vram_mib: 5632}`
|
|
9
|
-
# corrige só esse campo e herda o resto. É assim que um descriptor calibrado
|
|
10
|
-
# (`vramd calibrate --out`) entra em vigor sem editar nada aqui.
|
|
11
|
-
#
|
|
12
|
-
# Campos:
|
|
13
|
-
# name identificador do backend
|
|
14
|
-
# adapter dotted path do módulo com a classe `Adapter` (só modo in-process)
|
|
15
|
-
# vram_mib pico estimado; substituído pelo bloco `vram:` quando medido
|
|
16
|
-
# priority evicção: maior = manter carregado; menor = evictado primeiro
|
|
17
|
-
# runtime como arrancar o worker (command/cwd/env/timeouts)
|
|
18
|
-
# load_keys kwargs do request que influenciam a carga deste backend
|
|
19
|
-
# shape_keys subconjunto que força reload quando muda
|
|
20
|
-
# vram MEDIDO por `vramd calibrate` — vence a estimativa no admit
|
|
21
|
-
# peak_profile quant sob a qual foi medido + sinais (staged, unload_frees_vram)
|
|
22
|
-
|
|
23
|
-
version: 2
|
|
24
|
-
backends:
|
|
25
|
-
# Exemplo: modelo com venv próprio e comando explícito. É a forma canónica
|
|
26
|
-
# para qualquer backend fora deste repositório.
|
|
27
|
-
- name: example-whisper
|
|
28
|
-
adapter: my_whisper.adapter
|
|
29
|
-
vram_mib: 4200
|
|
30
|
-
priority: 20
|
|
31
|
-
runtime:
|
|
32
|
-
command: ["${env:WHISPER_PYTHON}", "-m", "my_whisper.worker"]
|
|
33
|
-
cwd: ~/models/whisper
|
|
34
|
-
env:
|
|
35
|
-
HF_HOME: ~/hf-cache
|
|
36
|
-
load_timeout_sec: 180
|
|
37
|
-
load_keys: [device, compute_type, beam_size]
|
|
38
|
-
shape_keys: [device, compute_type]
|
|
39
|
-
|
|
40
|
-
# Exemplo: atalho `tool:` — o interpretador sai de
|
|
41
|
-
# $VRAMD_TOOLS_ROOT/<tool>/.venv/bin/python e o comando é
|
|
42
|
-
# `python -m <tool> serve --vramd-worker`.
|
|
43
|
-
- name: example-diffusion
|
|
44
|
-
adapter: my_diffusion.adapter
|
|
45
|
-
vram_mib: 6000
|
|
46
|
-
priority: 40
|
|
47
|
-
tool: my_diffusion
|
|
48
|
-
footprint_key: flux-dev-uint4
|
|
49
|
-
peak_profile:
|
|
50
|
-
memory_efficient_with_quant: true
|
|
51
|
-
group_offload_with_memory_efficient: true
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|