simhall 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simhall/__init__.py +32 -0
- simhall/__main__.py +7 -0
- simhall/actions.py +103 -0
- simhall/cli.py +291 -0
- simhall/emulators/__init__.py +3 -0
- simhall/emulators/dcgm.py +99 -0
- simhall/emulators/lldp.py +188 -0
- simhall/emulators/netbox.py +709 -0
- simhall/emulators/redfish.py +395 -0
- simhall/faults.py +169 -0
- simhall/golden.py +92 -0
- simhall/hall.py +1066 -0
- simhall/models.py +269 -0
- simhall/naming.py +93 -0
- simhall/py.typed +0 -0
- simhall/seed_netbox.py +343 -0
- simhall/server.py +427 -0
- simhall/specfile.py +80 -0
- simhall/technician.py +567 -0
- simhall/views.py +123 -0
- simhall-0.1.0.dist-info/METADATA +133 -0
- simhall-0.1.0.dist-info/RECORD +26 -0
- simhall-0.1.0.dist-info/WHEEL +5 -0
- simhall-0.1.0.dist-info/entry_points.txt +2 -0
- simhall-0.1.0.dist-info/licenses/LICENSE +201 -0
- simhall-0.1.0.dist-info/top_level.txt +1 -0
simhall/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Copyright 2026 Hasanat Kazmi
|
|
3
|
+
"""simhall: a virtual GPU hall for testing bring-up tooling without hardware.
|
|
4
|
+
|
|
5
|
+
```python
|
|
6
|
+
from simhall import HallSpec, Hall, Fault, Action
|
|
7
|
+
hall = Hall.generate(HallSpec(site="hall-b"))
|
|
8
|
+
hall.inject(Fault(kind="wrong_port", cable="CBL-12-07-03", to_device="leaf4", to_interface="swp22"))
|
|
9
|
+
```
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
__version__ = "0.1.0"
|
|
15
|
+
|
|
16
|
+
from simhall.actions import Action
|
|
17
|
+
from simhall.faults import Fault
|
|
18
|
+
from simhall.hall import Conflict, Hall, SimError, UnknownObject
|
|
19
|
+
from simhall.models import CableSpec, Endpoint, HallSpec
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"Action",
|
|
23
|
+
"CableSpec",
|
|
24
|
+
"Conflict",
|
|
25
|
+
"Endpoint",
|
|
26
|
+
"Fault",
|
|
27
|
+
"Hall",
|
|
28
|
+
"HallSpec",
|
|
29
|
+
"SimError",
|
|
30
|
+
"UnknownObject",
|
|
31
|
+
"__version__",
|
|
32
|
+
]
|
simhall/__main__.py
ADDED
simhall/actions.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Copyright 2026 Hasanat Kazmi
|
|
3
|
+
"""Actions: what a technician (or a bot) physically does to the hall (CONTRACT.md section 6.2).
|
|
4
|
+
|
|
5
|
+
Actions change the ground truth and retire any fault they physically resolve; emulators reflect the
|
|
6
|
+
result on the next poll.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Literal
|
|
12
|
+
|
|
13
|
+
from pydantic import BaseModel, ConfigDict, model_validator
|
|
14
|
+
|
|
15
|
+
ActionKind = Literal[
|
|
16
|
+
"seat_cable",
|
|
17
|
+
"unseat_cable",
|
|
18
|
+
"power_on",
|
|
19
|
+
"power_off",
|
|
20
|
+
"replace_node",
|
|
21
|
+
"update_firmware",
|
|
22
|
+
"scan_qr",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
ACTION_KINDS: tuple[str, ...] = ActionKind.__args__ # type: ignore[attr-defined]
|
|
26
|
+
|
|
27
|
+
REQUIRED_FIELDS: dict[str, tuple[str, ...]] = {
|
|
28
|
+
"seat_cable": ("label",),
|
|
29
|
+
"unseat_cable": ("label",),
|
|
30
|
+
"power_on": ("device",),
|
|
31
|
+
"power_off": ("device",),
|
|
32
|
+
"replace_node": ("device", "new_serial"),
|
|
33
|
+
"update_firmware": ("device", "component", "version"),
|
|
34
|
+
"scan_qr": ("label",),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Action(BaseModel):
|
|
39
|
+
"""One physical action. For `seat_cable`, ends omitted default to the intent ends of `label`."""
|
|
40
|
+
|
|
41
|
+
model_config = ConfigDict(extra="forbid")
|
|
42
|
+
|
|
43
|
+
kind: ActionKind
|
|
44
|
+
label: str | None = None
|
|
45
|
+
a_device: str | None = None
|
|
46
|
+
a_interface: str | None = None
|
|
47
|
+
b_device: str | None = None
|
|
48
|
+
b_interface: str | None = None
|
|
49
|
+
device: str | None = None
|
|
50
|
+
new_serial: str | None = None
|
|
51
|
+
component: str | None = None
|
|
52
|
+
version: str | None = None
|
|
53
|
+
actor: str | None = None
|
|
54
|
+
|
|
55
|
+
@model_validator(mode="after")
|
|
56
|
+
def _required(self) -> Action:
|
|
57
|
+
missing = [f for f in REQUIRED_FIELDS[self.kind] if getattr(self, f) is None]
|
|
58
|
+
if missing:
|
|
59
|
+
raise ValueError(f"action {self.kind} requires {', '.join(missing)}")
|
|
60
|
+
return self
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def seat_cable(
|
|
64
|
+
label: str,
|
|
65
|
+
a_device: str | None = None,
|
|
66
|
+
a_interface: str | None = None,
|
|
67
|
+
b_device: str | None = None,
|
|
68
|
+
b_interface: str | None = None,
|
|
69
|
+
actor: str | None = None,
|
|
70
|
+
) -> Action:
|
|
71
|
+
return Action(
|
|
72
|
+
kind="seat_cable",
|
|
73
|
+
label=label,
|
|
74
|
+
a_device=a_device,
|
|
75
|
+
a_interface=a_interface,
|
|
76
|
+
b_device=b_device,
|
|
77
|
+
b_interface=b_interface,
|
|
78
|
+
actor=actor,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def unseat_cable(label: str, actor: str | None = None) -> Action:
|
|
83
|
+
return Action(kind="unseat_cable", label=label, actor=actor)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def power_on(device: str, actor: str | None = None) -> Action:
|
|
87
|
+
return Action(kind="power_on", device=device, actor=actor)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def power_off(device: str, actor: str | None = None) -> Action:
|
|
91
|
+
return Action(kind="power_off", device=device, actor=actor)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def replace_node(device: str, new_serial: str, actor: str | None = None) -> Action:
|
|
95
|
+
return Action(kind="replace_node", device=device, new_serial=new_serial, actor=actor)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def update_firmware(device: str, component: str, version: str, actor: str | None = None) -> Action:
|
|
99
|
+
return Action(kind="update_firmware", device=device, component=component, version=version, actor=actor)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def scan_qr(label: str, actor: str | None = None) -> Action:
|
|
103
|
+
return Action(kind="scan_qr", label=label, actor=actor)
|
simhall/cli.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Copyright 2026 Hasanat Kazmi
|
|
3
|
+
"""`simhall` command line: serve, intent, golden-spec, technician, seed-netbox, fault, act, faults, reset."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import json
|
|
9
|
+
import sys
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import click
|
|
14
|
+
import httpx
|
|
15
|
+
import yaml
|
|
16
|
+
|
|
17
|
+
from simhall import __version__
|
|
18
|
+
from simhall.golden import GOLDEN_NAMES, build_golden, golden_cables, golden_faults, golden_spec
|
|
19
|
+
from simhall.hall import Hall
|
|
20
|
+
from simhall.specfile import SpecFile, dump_spec_file, load_spec_file
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _builder(
|
|
24
|
+
golden: str | None, spec_path: str | None, base_url: str | None = None
|
|
25
|
+
) -> tuple[Callable[[], Hall], str]:
|
|
26
|
+
"""A hall builder from `--golden` or `--spec`; the golden `hall-b` when neither is given."""
|
|
27
|
+
if golden and spec_path:
|
|
28
|
+
raise click.UsageError("use either --golden or --spec, not both")
|
|
29
|
+
if spec_path:
|
|
30
|
+
spec_file = load_spec_file(spec_path)
|
|
31
|
+
return (lambda: spec_file.build(base_url)), f"spec {spec_path}"
|
|
32
|
+
name = golden or "hall-b"
|
|
33
|
+
if name not in GOLDEN_NAMES:
|
|
34
|
+
raise click.UsageError(f"unknown golden scenario {name!r}; known: {', '.join(GOLDEN_NAMES)}")
|
|
35
|
+
return (lambda: build_golden(name, base_url)), f"golden {name}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@click.group()
|
|
39
|
+
@click.version_option(__version__, prog_name="simhall")
|
|
40
|
+
def main() -> None:
|
|
41
|
+
"""simhall: a virtual GPU hall (Redfish, LLDP switches, DCGM, NetBox) with fault injection."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@main.command()
|
|
45
|
+
@click.option(
|
|
46
|
+
"--golden",
|
|
47
|
+
default=None,
|
|
48
|
+
help=f"Built-in scenario ({', '.join(GOLDEN_NAMES)}). Default when --spec is absent.",
|
|
49
|
+
)
|
|
50
|
+
@click.option(
|
|
51
|
+
"--spec", "spec_path", type=click.Path(exists=True, dir_okay=False), default=None, help="Spec YAML/JSON."
|
|
52
|
+
)
|
|
53
|
+
@click.option("--host", default="127.0.0.1", show_default=True)
|
|
54
|
+
@click.option("--port", default=8900, show_default=True, type=int)
|
|
55
|
+
@click.option(
|
|
56
|
+
"--timeout-sleep", default=10.0, show_default=True, type=float, help="Seconds a 'timeout' device stalls."
|
|
57
|
+
)
|
|
58
|
+
@click.option("--log-level", default="info", show_default=True)
|
|
59
|
+
def serve(
|
|
60
|
+
golden: str | None, spec_path: str | None, host: str, port: int, timeout_sleep: float, log_level: str
|
|
61
|
+
) -> None:
|
|
62
|
+
"""Serve the hall on 127.0.0.1 (vendor protocols, NetBox API and the /sim control plane)."""
|
|
63
|
+
from simhall.server import serve as _serve # noqa: PLC0415 - keeps `simhall --help` fast
|
|
64
|
+
|
|
65
|
+
builder, label = _builder(golden, spec_path, f"http://{host}:{port}")
|
|
66
|
+
click.echo(f"simhall {__version__}: serving {label} on http://{host}:{port}", err=True)
|
|
67
|
+
click.echo(f" NetBox API NETBOX_URL=http://{host}:{port}/netbox", err=True)
|
|
68
|
+
click.echo(f" intent http://{host}:{port}/sim/intent", err=True)
|
|
69
|
+
click.echo(f" control plane http://{host}:{port}/sim/faults /sim/actions /sim/reset", err=True)
|
|
70
|
+
_serve(builder, host=host, port=port, timeout_sleep_s=timeout_sleep, log_level=log_level)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@main.command()
|
|
74
|
+
@click.option("--golden", default=None, help="Built-in scenario.")
|
|
75
|
+
@click.option("--spec", "spec_path", type=click.Path(exists=True, dir_okay=False), default=None)
|
|
76
|
+
@click.option(
|
|
77
|
+
"--base-url", default=None, help="mgmt URL prefix (default: the spec's base_url, http://127.0.0.1:8900)."
|
|
78
|
+
)
|
|
79
|
+
@click.option(
|
|
80
|
+
"-o",
|
|
81
|
+
"--out",
|
|
82
|
+
type=click.Path(dir_okay=False),
|
|
83
|
+
default=None,
|
|
84
|
+
help="Write here (.yaml/.yml or .json) instead of stdout.",
|
|
85
|
+
)
|
|
86
|
+
def intent(golden: str | None, spec_path: str | None, base_url: str | None, out: str | None) -> None:
|
|
87
|
+
"""Print or write the turnup.intent/v1 document (simulated: true)."""
|
|
88
|
+
builder, _ = _builder(golden, spec_path, base_url)
|
|
89
|
+
hall = builder()
|
|
90
|
+
if out:
|
|
91
|
+
path = hall.write_intent(out)
|
|
92
|
+
click.echo(
|
|
93
|
+
f"wrote {path} ({len(hall.intent_cables)} cables, {len(hall.nodes)} nodes, "
|
|
94
|
+
f"{len(hall.leaves)} leaves)",
|
|
95
|
+
err=True,
|
|
96
|
+
)
|
|
97
|
+
else:
|
|
98
|
+
click.echo(yaml.safe_dump(hall.intent(), sort_keys=False, default_flow_style=False), nl=False)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@main.command("golden-spec")
|
|
102
|
+
@click.argument("name", default="hall-b")
|
|
103
|
+
@click.option("-o", "--out", type=click.Path(dir_okay=False), default=None)
|
|
104
|
+
def golden_spec_cmd(name: str, out: str | None) -> None:
|
|
105
|
+
"""Export a golden scenario as a spec file (explicit cables and faults included)."""
|
|
106
|
+
if name not in GOLDEN_NAMES:
|
|
107
|
+
raise click.UsageError(f"unknown golden scenario {name!r}; known: {', '.join(GOLDEN_NAMES)}")
|
|
108
|
+
spec = golden_spec(name)
|
|
109
|
+
spec_file = SpecFile(spec=spec, cables=golden_cables(spec), faults=golden_faults())
|
|
110
|
+
text = dump_spec_file(spec_file, out)
|
|
111
|
+
if out:
|
|
112
|
+
click.echo(f"wrote {out}", err=True)
|
|
113
|
+
else:
|
|
114
|
+
click.echo(text, nl=False)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@main.command()
|
|
118
|
+
@click.option("--acceptance-url", default="http://127.0.0.1:8800", show_default=True)
|
|
119
|
+
@click.option(
|
|
120
|
+
"--sim-url", default="http://127.0.0.1:8900", show_default=True, help="simhall server to act on."
|
|
121
|
+
)
|
|
122
|
+
@click.option("--worker", default="tech-1", show_default=True)
|
|
123
|
+
@click.option("--error-rate", default=0.05, show_default=True, type=float)
|
|
124
|
+
@click.option(
|
|
125
|
+
"--speed",
|
|
126
|
+
default=20.0,
|
|
127
|
+
show_default=True,
|
|
128
|
+
type=float,
|
|
129
|
+
help="Simulation speed factor (20 = a 60 s step takes 3 s).",
|
|
130
|
+
)
|
|
131
|
+
@click.option("--rack", default=None, help="Only pull steps for this rack.")
|
|
132
|
+
@click.option("--seed", default=None, type=int, help="Seed for reproducible mistakes.")
|
|
133
|
+
@click.option(
|
|
134
|
+
"--approver", default=None, help="If set, approve requires_approval steps as this named person."
|
|
135
|
+
)
|
|
136
|
+
@click.option("--max-steps", default=None, type=int)
|
|
137
|
+
@click.option(
|
|
138
|
+
"--idle-polls", default=5, show_default=True, type=int, help="Consecutive empty polls before exiting."
|
|
139
|
+
)
|
|
140
|
+
@click.option(
|
|
141
|
+
"--poll-interval", default=1.0, show_default=True, type=float, help="Seconds between empty polls."
|
|
142
|
+
)
|
|
143
|
+
@click.option("--json", "as_json", is_flag=True, help="Print the stats as JSON.")
|
|
144
|
+
def technician(
|
|
145
|
+
acceptance_url: str,
|
|
146
|
+
sim_url: str,
|
|
147
|
+
worker: str,
|
|
148
|
+
error_rate: float,
|
|
149
|
+
speed: float,
|
|
150
|
+
rack: str | None,
|
|
151
|
+
seed: int | None,
|
|
152
|
+
approver: str | None,
|
|
153
|
+
max_steps: int | None,
|
|
154
|
+
idle_polls: int,
|
|
155
|
+
poll_interval: float,
|
|
156
|
+
as_json: bool,
|
|
157
|
+
) -> None:
|
|
158
|
+
"""Run a simulated technician against the Acceptance API, acting on the simhall server."""
|
|
159
|
+
from simhall.technician import run_bot # noqa: PLC0415
|
|
160
|
+
|
|
161
|
+
stats = asyncio.run(
|
|
162
|
+
run_bot(
|
|
163
|
+
acceptance_url,
|
|
164
|
+
sim_url=sim_url,
|
|
165
|
+
worker=worker,
|
|
166
|
+
error_rate=error_rate,
|
|
167
|
+
speed=speed,
|
|
168
|
+
rack=rack,
|
|
169
|
+
seed=seed,
|
|
170
|
+
approver=approver,
|
|
171
|
+
max_steps=max_steps,
|
|
172
|
+
idle_polls=idle_polls,
|
|
173
|
+
poll_interval_s=poll_interval,
|
|
174
|
+
)
|
|
175
|
+
)
|
|
176
|
+
if as_json:
|
|
177
|
+
click.echo(json.dumps(stats.as_dict(), indent=2))
|
|
178
|
+
else:
|
|
179
|
+
d = stats.as_dict()
|
|
180
|
+
click.echo(
|
|
181
|
+
f"{worker}: {d['done']} steps done, {d['mistakes']} mistakes "
|
|
182
|
+
f"({d['wrong_port']} wrong port, {d['unseated']} unseated), {d['errors']} errors"
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@main.command("seed-netbox")
|
|
187
|
+
@click.option("--netbox-url", required=True, envvar="NETBOX_URL", help="e.g. http://127.0.0.1:8000")
|
|
188
|
+
@click.option("--token", required=False, envvar="NETBOX_TOKEN", default=None)
|
|
189
|
+
@click.option("--golden", default=None)
|
|
190
|
+
@click.option("--spec", "spec_path", type=click.Path(exists=True, dir_okay=False), default=None)
|
|
191
|
+
@click.option(
|
|
192
|
+
"--intent",
|
|
193
|
+
"intent_path",
|
|
194
|
+
type=click.Path(exists=True, dir_okay=False),
|
|
195
|
+
default=None,
|
|
196
|
+
help="Seed from an intent file instead.",
|
|
197
|
+
)
|
|
198
|
+
@click.option(
|
|
199
|
+
"--base-url",
|
|
200
|
+
default="http://127.0.0.1:8900",
|
|
201
|
+
show_default=True,
|
|
202
|
+
help="mgmt URL prefix written into custom fields.",
|
|
203
|
+
)
|
|
204
|
+
@click.option("--dry-run", is_flag=True, help="Print the plan; do not touch NetBox (no pynetbox import).")
|
|
205
|
+
def seed_netbox_cmd(
|
|
206
|
+
netbox_url: str,
|
|
207
|
+
token: str | None,
|
|
208
|
+
golden: str | None,
|
|
209
|
+
spec_path: str | None,
|
|
210
|
+
intent_path: str | None,
|
|
211
|
+
base_url: str,
|
|
212
|
+
dry_run: bool,
|
|
213
|
+
) -> None:
|
|
214
|
+
"""Seed a real NetBox (netbox-docker) with the hall via pynetbox."""
|
|
215
|
+
from simhall.seed_netbox import connect, plan_from_intent, seed # noqa: PLC0415
|
|
216
|
+
|
|
217
|
+
if intent_path:
|
|
218
|
+
text = Path(intent_path).read_text(encoding="utf-8")
|
|
219
|
+
doc = json.loads(text) if intent_path.endswith(".json") else yaml.safe_load(text)
|
|
220
|
+
else:
|
|
221
|
+
builder, _ = _builder(golden, spec_path, base_url)
|
|
222
|
+
doc = builder().intent()
|
|
223
|
+
if dry_run:
|
|
224
|
+
plan = plan_from_intent(doc)
|
|
225
|
+
click.echo(f"dry run against {netbox_url}: would create {json.dumps(plan.counts())}")
|
|
226
|
+
for d in plan.devices[:3]:
|
|
227
|
+
click.echo(
|
|
228
|
+
f" device {d['name']} ({d['role']}, {d['device_type']}) "
|
|
229
|
+
f"custom_fields={json.dumps(d['custom_fields'])}"
|
|
230
|
+
)
|
|
231
|
+
for c in plan.cables[:3]:
|
|
232
|
+
click.echo(
|
|
233
|
+
f" cable {c['label']}: {c['a']['device']} {c['a']['interface']} "
|
|
234
|
+
f"-> {c['b']['device']} {c['b']['interface']}"
|
|
235
|
+
)
|
|
236
|
+
return
|
|
237
|
+
if not token:
|
|
238
|
+
raise click.UsageError("--token (or NETBOX_TOKEN) is required unless --dry-run")
|
|
239
|
+
nb = connect(netbox_url, token)
|
|
240
|
+
summary = seed(doc, nb, log=lambda m: click.echo(m, err=True))
|
|
241
|
+
click.echo(json.dumps({k: v for k, v in summary.items() if k != "created_items"}))
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _control(url: str, method: str, path: str, body: str | None = None) -> None:
|
|
245
|
+
payload = json.loads(body) if body else None
|
|
246
|
+
with httpx.Client(timeout=10.0) as client:
|
|
247
|
+
r = client.request(method, f"{url.rstrip('/')}{path}", json=payload)
|
|
248
|
+
try:
|
|
249
|
+
click.echo(json.dumps(r.json(), indent=2))
|
|
250
|
+
except ValueError:
|
|
251
|
+
click.echo(r.text)
|
|
252
|
+
if r.status_code >= 400:
|
|
253
|
+
sys.exit(1)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
@main.command()
|
|
257
|
+
@click.option("--url", default="http://127.0.0.1:8900", show_default=True)
|
|
258
|
+
@click.argument("fault_json")
|
|
259
|
+
def fault(url: str, fault_json: str) -> None:
|
|
260
|
+
"""Inject a fault from JSON, e.g. '{"kind":"missing_link","cable":"CBL-12-07-03"}'."""
|
|
261
|
+
_control(url, "POST", "/sim/faults", fault_json)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
@main.command()
|
|
265
|
+
@click.option("--url", default="http://127.0.0.1:8900", show_default=True)
|
|
266
|
+
@click.argument("action_json")
|
|
267
|
+
def act(url: str, action_json: str) -> None:
|
|
268
|
+
"""Perform an action: simhall act '{"kind":"seat_cable","label":"CBL-12-07-03","b_interface":"swp21"}'."""
|
|
269
|
+
_control(url, "POST", "/sim/actions", action_json)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@main.command()
|
|
273
|
+
@click.option("--url", default="http://127.0.0.1:8900", show_default=True)
|
|
274
|
+
@click.option("--clear", "clear_id", default=None, help="Clear this fault id instead of listing.")
|
|
275
|
+
def faults(url: str, clear_id: str | None) -> None:
|
|
276
|
+
"""List active faults, or clear one with --clear ID."""
|
|
277
|
+
if clear_id:
|
|
278
|
+
_control(url, "DELETE", f"/sim/faults/{clear_id}")
|
|
279
|
+
else:
|
|
280
|
+
_control(url, "GET", "/sim/faults")
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
@main.command()
|
|
284
|
+
@click.option("--url", default="http://127.0.0.1:8900", show_default=True)
|
|
285
|
+
def reset(url: str) -> None:
|
|
286
|
+
"""Rebuild the hall as served at start-up (the golden fault comes back)."""
|
|
287
|
+
_control(url, "POST", "/sim/reset")
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
if __name__ == "__main__": # pragma: no cover
|
|
291
|
+
main()
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Copyright 2026 Hasanat Kazmi
|
|
3
|
+
"""DCGM-exporter Prometheus exposition text (public metric names, CONTRACT.md section 3.3)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from simhall.models import HallSpec, NodeIntent, NodeState, gpu_model_name
|
|
8
|
+
|
|
9
|
+
CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
|
|
10
|
+
DRIVER_VERSION = "580.82.07"
|
|
11
|
+
|
|
12
|
+
# (metric family, type, help) in the order dcgm-exporter prints its default set.
|
|
13
|
+
FAMILIES: tuple[tuple[str, str, str], ...] = (
|
|
14
|
+
("DCGM_FI_DEV_SM_CLOCK", "gauge", "SM clock frequency (in MHz)."),
|
|
15
|
+
("DCGM_FI_DEV_GPU_TEMP", "gauge", "GPU temperature (in C)."),
|
|
16
|
+
("DCGM_FI_DEV_POWER_USAGE", "gauge", "Power draw (in W)."),
|
|
17
|
+
("DCGM_FI_DEV_FB_TOTAL", "gauge", "Framebuffer memory total (in MiB)."),
|
|
18
|
+
("DCGM_FI_DEV_PCIE_REPLAY_COUNTER", "counter", "Total number of PCIe retries."),
|
|
19
|
+
("DCGM_FI_DEV_XID_ERRORS", "gauge", "Value of the last XID error encountered."),
|
|
20
|
+
("DCGM_FI_DEV_ECC_DBE_VOL_TOTAL", "counter", "Total number of double-bit volatile ECC errors."),
|
|
21
|
+
("DCGM_FI_DEV_ROW_REMAP_FAILURE", "gauge", "Whether remapping of rows has failed."),
|
|
22
|
+
(
|
|
23
|
+
"DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS",
|
|
24
|
+
"counter",
|
|
25
|
+
"Number of remapped rows for uncorrectable errors.",
|
|
26
|
+
),
|
|
27
|
+
(
|
|
28
|
+
"DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL",
|
|
29
|
+
"counter",
|
|
30
|
+
"Total number of NVLink flow-control CRC errors.",
|
|
31
|
+
),
|
|
32
|
+
(
|
|
33
|
+
"DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL",
|
|
34
|
+
"counter",
|
|
35
|
+
"Total number of NVLink bandwidth counters for all lanes.",
|
|
36
|
+
),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
NVLINK_DEGRADED_CRC_ERRORS = 1500
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _labels(nd: NodeIntent, model_name: str, g: int, gpu_uuid: str) -> str:
|
|
43
|
+
bus = f"00000000:{0x0F + 0x10 * g:02X}:00.0"
|
|
44
|
+
return (
|
|
45
|
+
f'gpu="{g}",UUID="{gpu_uuid}",pci_bus_id="{bus}",device="nvidia{g}",'
|
|
46
|
+
f'modelName="{model_name}",Hostname="{nd.name}",DCGM_FI_DRIVER_VERSION="{DRIVER_VERSION}"'
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def render(spec: HallSpec, nd: NodeIntent, ns: NodeState) -> str:
|
|
51
|
+
"""The `/metrics` body: one sample per present GPU per family."""
|
|
52
|
+
model_name = gpu_model_name(spec.node_model)
|
|
53
|
+
present = [g for g in ns.gpus if g.present]
|
|
54
|
+
lines: list[str] = []
|
|
55
|
+
for family, kind, help_text in FAMILIES:
|
|
56
|
+
lines.append(f"# HELP {family} {help_text}")
|
|
57
|
+
lines.append(f"# TYPE {family} {kind}")
|
|
58
|
+
for g in present:
|
|
59
|
+
labels = _labels(nd, model_name, g.index, g.uuid)
|
|
60
|
+
value: int | float
|
|
61
|
+
if family == "DCGM_FI_DEV_SM_CLOCK":
|
|
62
|
+
value = g.sm_clock_mhz
|
|
63
|
+
elif family == "DCGM_FI_DEV_GPU_TEMP":
|
|
64
|
+
value = g.temp_c
|
|
65
|
+
elif family == "DCGM_FI_DEV_POWER_USAGE":
|
|
66
|
+
value = g.power_w
|
|
67
|
+
elif family == "DCGM_FI_DEV_FB_TOTAL":
|
|
68
|
+
value = g.fb_total_mib
|
|
69
|
+
elif family == "DCGM_FI_DEV_PCIE_REPLAY_COUNTER":
|
|
70
|
+
value = g.pcie_replay
|
|
71
|
+
elif family == "DCGM_FI_DEV_XID_ERRORS":
|
|
72
|
+
value = g.xid_errors
|
|
73
|
+
elif family == "DCGM_FI_DEV_ECC_DBE_VOL_TOTAL":
|
|
74
|
+
value = g.ecc_dbe
|
|
75
|
+
elif family == "DCGM_FI_DEV_ROW_REMAP_FAILURE":
|
|
76
|
+
value = g.row_remap_failure
|
|
77
|
+
elif family == "DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS":
|
|
78
|
+
value = g.uncorrectable_remapped_rows
|
|
79
|
+
elif family == "DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL":
|
|
80
|
+
value = (
|
|
81
|
+
(NVLINK_DEGRADED_CRC_ERRORS + 37 * g.index) if ns.nvlink_degraded else g.nvlink_crc_errors
|
|
82
|
+
)
|
|
83
|
+
else: # DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL
|
|
84
|
+
value = g.nvlink_bandwidth // 4 if ns.nvlink_degraded else g.nvlink_bandwidth
|
|
85
|
+
lines.append(f"{family}{{{labels}}} {value}")
|
|
86
|
+
return "\n".join(lines) + "\n"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def parse(text: str) -> dict[str, dict[str, float]]:
|
|
90
|
+
"""Tiny exposition parser for tests: family -> {gpu label -> value}."""
|
|
91
|
+
out: dict[str, dict[str, float]] = {}
|
|
92
|
+
for line in text.splitlines():
|
|
93
|
+
if not line or line.startswith("#"):
|
|
94
|
+
continue
|
|
95
|
+
name, rest = line.split("{", 1)
|
|
96
|
+
labels, value = rest.rsplit("} ", 1)
|
|
97
|
+
gpu = labels.split('gpu="', 1)[1].split('"', 1)[0]
|
|
98
|
+
out.setdefault(name, {})[gpu] = float(value)
|
|
99
|
+
return out
|