proxlb-solver 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proxlb_solver/__init__.py +53 -0
- proxlb_solver/adapter.py +332 -0
- proxlb_solver/cli.py +116 -0
- proxlb_solver/loader.py +181 -0
- proxlb_solver/models.py +205 -0
- proxlb_solver/planner.py +388 -0
- proxlb_solver/py.typed +0 -0
- proxlb_solver/reporter.py +967 -0
- proxlb_solver/shadow.py +743 -0
- proxlb_solver/shadow_reporter.py +1313 -0
- proxlb_solver/simulate.py +34 -0
- proxlb_solver/solver.py +479 -0
- proxlb_solver/validator.py +111 -0
- proxlb_solver-0.1.0.dist-info/METADATA +234 -0
- proxlb_solver-0.1.0.dist-info/RECORD +18 -0
- proxlb_solver-0.1.0.dist-info/WHEEL +5 -0
- proxlb_solver-0.1.0.dist-info/entry_points.txt +3 -0
- proxlb_solver-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""ProxLB CP-SAT VM Scheduler.
|
|
2
|
+
|
|
3
|
+
Public API
|
|
4
|
+
----------
|
|
5
|
+
Core types:
|
|
6
|
+
Cluster, Node, VM, Constraints, Balancing,
|
|
7
|
+
Solution, Migration, MigrationPlan, SolverStats
|
|
8
|
+
|
|
9
|
+
Solver functions:
|
|
10
|
+
solve(cluster, time_limit_s) — single-shot solve
|
|
11
|
+
solve_reachable(cluster, ...) — solve with planner feedback loop
|
|
12
|
+
|
|
13
|
+
ProxLB integration:
|
|
14
|
+
from_proxlb_data(proxlb_data, ...) — convert live ProxLB data to Cluster
|
|
15
|
+
run_shadow(proxlb_data, cfg) — shadow-mode observer (read-only)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
__version__ = "0.1.0"
|
|
19
|
+
|
|
20
|
+
from .models import (
|
|
21
|
+
Cluster,
|
|
22
|
+
Node,
|
|
23
|
+
VM,
|
|
24
|
+
Constraints,
|
|
25
|
+
Balancing,
|
|
26
|
+
Solution,
|
|
27
|
+
Migration,
|
|
28
|
+
MigrationPlan,
|
|
29
|
+
SolverStats,
|
|
30
|
+
)
|
|
31
|
+
from .solver import solve, solve_reachable
|
|
32
|
+
from .adapter import from_proxlb_data
|
|
33
|
+
from .shadow import run_shadow
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"__version__",
|
|
37
|
+
# Data models
|
|
38
|
+
"Cluster",
|
|
39
|
+
"Node",
|
|
40
|
+
"VM",
|
|
41
|
+
"Constraints",
|
|
42
|
+
"Balancing",
|
|
43
|
+
"Solution",
|
|
44
|
+
"Migration",
|
|
45
|
+
"MigrationPlan",
|
|
46
|
+
"SolverStats",
|
|
47
|
+
# Solver
|
|
48
|
+
"solve",
|
|
49
|
+
"solve_reachable",
|
|
50
|
+
# ProxLB integration
|
|
51
|
+
"from_proxlb_data",
|
|
52
|
+
"run_shadow",
|
|
53
|
+
]
|
proxlb_solver/adapter.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Adapter to convert ProxLB data into Solver models.
|
|
3
|
+
|
|
4
|
+
Accepts either:
|
|
5
|
+
- a plain dict (standalone CLI use, unit tests, ProxLB ≤ 1.x)
|
|
6
|
+
- a ProxLbData Pydantic model (ProxLB 2.0 integration)
|
|
7
|
+
|
|
8
|
+
When a Pydantic model is received it is first normalised to the same flat-dict
|
|
9
|
+
layout that the rest of this module expects, so there is a single code path
|
|
10
|
+
after normalisation.
|
|
11
|
+
|
|
12
|
+
Memory / reservation handling
|
|
13
|
+
------------------------------
|
|
14
|
+
ProxLB stores the *already-reduced* value in node memory_total after applying
|
|
15
|
+
reservations. The raw hardware total is reconstructed as:
|
|
16
|
+
|
|
17
|
+
raw_memory = memory_used + memory_free (both come straight from the PVE API)
|
|
18
|
+
|
|
19
|
+
By default (use_reservations=True) this adapter reconstructs the raw total and
|
|
20
|
+
re-applies the reservation explicitly in Node.memory_reserve so the solver's
|
|
21
|
+
constraints are transparent. With use_reservations=False the reservation is
|
|
22
|
+
zero, letting the solver use the full hardware capacity.
|
|
23
|
+
|
|
24
|
+
Fallback: if memory_used + memory_free == 0 (e.g. synthetic test data),
|
|
25
|
+
memory_total is used as-is with memory_reserve=0.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
from collections import defaultdict
|
|
31
|
+
from typing import TYPE_CHECKING, Any, Dict
|
|
32
|
+
|
|
33
|
+
from .models import Balancing, Cluster, Constraints, Expect, Node, VM
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from proxlb.utils.proxlb_data import ProxLbData
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ---------------------------------------------------------------------------
|
|
40
|
+
# Normalisation: Pydantic → flat dict
|
|
41
|
+
# ---------------------------------------------------------------------------
|
|
42
|
+
|
|
43
|
+
def _reserve_to_dict(raw: Any) -> dict[str, Any]:
|
|
44
|
+
"""Convert node_resource_reserve (may have StrEnum keys) to plain str keys."""
|
|
45
|
+
if not raw:
|
|
46
|
+
return {}
|
|
47
|
+
result = {}
|
|
48
|
+
for node_name, res_dict in raw.items():
|
|
49
|
+
result[str(node_name)] = {str(k): v for k, v in res_dict.items()}
|
|
50
|
+
return result
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _pydantic_to_dict(proxlb_data: "ProxLbData") -> Dict[str, Any]:
|
|
54
|
+
"""
|
|
55
|
+
Flatten a ProxLbData Pydantic model into the dict layout expected by
|
|
56
|
+
from_proxlb_data(). Only the fields the adapter actually reads are mapped.
|
|
57
|
+
"""
|
|
58
|
+
bc = proxlb_data.meta.balancing
|
|
59
|
+
|
|
60
|
+
balancing_dict: Dict[str, Any] = {
|
|
61
|
+
"method": str(bc.method),
|
|
62
|
+
"mode": str(bc.mode),
|
|
63
|
+
"balanciness": bc.balanciness,
|
|
64
|
+
"cpu_overcommit": getattr(bc, "cpu_overcommit", 2.0),
|
|
65
|
+
"memory_threshold": bc.memory_threshold,
|
|
66
|
+
"cpu_threshold": bc.cpu_threshold,
|
|
67
|
+
"disk_threshold": bc.disk_threshold,
|
|
68
|
+
"max_node_inflow": getattr(bc, "max_node_inflow", 1),
|
|
69
|
+
"max_parallel_migrations": getattr(bc, "max_parallel_migrations", None),
|
|
70
|
+
"node_resource_reserve": _reserve_to_dict(bc.node_resource_reserve),
|
|
71
|
+
"pools": bc.pools,
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
nodes: Dict[str, Any] = {}
|
|
75
|
+
for name, nd in proxlb_data.nodes.items():
|
|
76
|
+
nodes[name] = {
|
|
77
|
+
"cpu_total": nd.cpu.total,
|
|
78
|
+
"memory_used": int(nd.memory.used),
|
|
79
|
+
"memory_free": int(nd.memory.free),
|
|
80
|
+
"memory_total": nd.memory.total,
|
|
81
|
+
"disk_free": int(nd.disk.free),
|
|
82
|
+
"cpu_pressure_some_percent": nd.cpu.pressure_some_percent,
|
|
83
|
+
"memory_pressure_some_percent": nd.memory.pressure_some_percent,
|
|
84
|
+
"disk_pressure_some_percent": nd.disk.pressure_some_percent,
|
|
85
|
+
"maintenance": nd.maintenance,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
guests: Dict[str, Any] = {}
|
|
89
|
+
for name, gd in proxlb_data.guests.items():
|
|
90
|
+
guests[name] = {
|
|
91
|
+
"node_current": gd.node_current,
|
|
92
|
+
"node_target": gd.node_target,
|
|
93
|
+
"cpu_total": gd.cpu.total,
|
|
94
|
+
"memory_total": gd.memory.total,
|
|
95
|
+
"cpu_used": gd.cpu.used,
|
|
96
|
+
"disk_used": int(gd.disk.used),
|
|
97
|
+
"cpu_pressure_some_percent": gd.cpu.pressure_some_percent,
|
|
98
|
+
"memory_pressure_some_percent": gd.memory.pressure_some_percent,
|
|
99
|
+
"disk_pressure_some_percent": gd.disk.pressure_some_percent,
|
|
100
|
+
"type": str(gd.type),
|
|
101
|
+
"ha_rules": [
|
|
102
|
+
{"rule": r.rule, "type": str(r.type), "nodes": r.nodes}
|
|
103
|
+
for r in gd.ha_rules
|
|
104
|
+
],
|
|
105
|
+
"tags": gd.tags,
|
|
106
|
+
"pools": gd.pools,
|
|
107
|
+
"node_relationships": gd.node_relationships,
|
|
108
|
+
"ignore": gd.ignore,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return {
|
|
112
|
+
"meta": {"balancing": balancing_dict, "cluster_name": "Live Cluster"},
|
|
113
|
+
"nodes": nodes,
|
|
114
|
+
"guests": guests,
|
|
115
|
+
"groups": {},
|
|
116
|
+
"ha_rules": {},
|
|
117
|
+
"pools": {},
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
# Internal helpers
|
|
123
|
+
# ---------------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
def _get_memory_reserve_bytes(node_name: str, reserve_cfg: dict[str, Any]) -> int:
|
|
126
|
+
"""
|
|
127
|
+
Returns the configured memory reservation for a specific node in bytes.
|
|
128
|
+
|
|
129
|
+
Resolution order:
|
|
130
|
+
1. Node-specific override (e.g. reserve_cfg['pve-01']['memory'])
|
|
131
|
+
2. Global default (reserve_cfg['defaults']['memory'])
|
|
132
|
+
3. Zero (nothing configured)
|
|
133
|
+
"""
|
|
134
|
+
node_config = reserve_cfg.get(node_name, {})
|
|
135
|
+
default_config = reserve_cfg.get("defaults", {})
|
|
136
|
+
gb = node_config.get("memory") or default_config.get("memory", 0)
|
|
137
|
+
if not isinstance(gb, (int, float)):
|
|
138
|
+
gb = 0
|
|
139
|
+
return int(gb * 1024 ** 3)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# ---------------------------------------------------------------------------
|
|
143
|
+
# Public API
|
|
144
|
+
# ---------------------------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
def from_proxlb_data(
|
|
147
|
+
proxlb_data: "Dict[str, Any] | ProxLbData",
|
|
148
|
+
use_reservations: bool = True,
|
|
149
|
+
pin_vms: set[str] | None = None,
|
|
150
|
+
) -> Cluster:
|
|
151
|
+
"""
|
|
152
|
+
Convert ProxLB data into a Cluster ready for the solver.
|
|
153
|
+
|
|
154
|
+
Accepts either a plain dict (standalone / tests) or a ProxLbData Pydantic
|
|
155
|
+
model (ProxLB 2.0 integration). Pydantic models are normalised to a flat
|
|
156
|
+
dict internally so only one code path exists after this point.
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
proxlb_data: Dict or ProxLbData from the ProxLB main loop.
|
|
160
|
+
use_reservations: If True, node memory reservations are applied explicitly.
|
|
161
|
+
pin_vms: VM names to force-pin to their current node (active-mode
|
|
162
|
+
retry: VMs that failed to migrate previously).
|
|
163
|
+
"""
|
|
164
|
+
if not isinstance(proxlb_data, dict):
|
|
165
|
+
proxlb_data = _pydantic_to_dict(proxlb_data)
|
|
166
|
+
|
|
167
|
+
meta = proxlb_data.get("meta", {})
|
|
168
|
+
balancing_cfg = meta.get("balancing", {}) if isinstance(meta, dict) else {}
|
|
169
|
+
|
|
170
|
+
# 1. Balancing configuration -------------------------------------------
|
|
171
|
+
balancing = Balancing(
|
|
172
|
+
method=balancing_cfg.get("method", "memory"),
|
|
173
|
+
mode=balancing_cfg.get("mode", "used"),
|
|
174
|
+
balanciness=balancing_cfg.get("balanciness", 3),
|
|
175
|
+
cpu_overcommit=balancing_cfg.get("cpu_overcommit", 2.0),
|
|
176
|
+
memory_threshold=balancing_cfg.get("memory_threshold"),
|
|
177
|
+
cpu_threshold=balancing_cfg.get("cpu_threshold"),
|
|
178
|
+
disk_threshold=balancing_cfg.get("disk_threshold"),
|
|
179
|
+
max_node_inflow=balancing_cfg.get("max_node_inflow", 1),
|
|
180
|
+
max_parallel_migrations=balancing_cfg.get("max_parallel_migrations"),
|
|
181
|
+
)
|
|
182
|
+
reserve_cfg: dict[str, Any] = balancing_cfg.get("node_resource_reserve") or {}
|
|
183
|
+
|
|
184
|
+
# 2. Physical nodes --------------------------------------------------------
|
|
185
|
+
nodes = []
|
|
186
|
+
for name, nd in proxlb_data.get("nodes", {}).items():
|
|
187
|
+
|
|
188
|
+
def _nd(key: str, default: Any = 0, _nd: dict[str, Any] = nd) -> Any: # noqa: E731
|
|
189
|
+
return _nd.get(key, default)
|
|
190
|
+
|
|
191
|
+
mem_used = int(_nd("memory_used"))
|
|
192
|
+
mem_free = int(_nd("memory_free"))
|
|
193
|
+
raw_memory = mem_used + mem_free
|
|
194
|
+
|
|
195
|
+
if raw_memory > 0:
|
|
196
|
+
memory_reserve = (
|
|
197
|
+
int(_get_memory_reserve_bytes(name, reserve_cfg))
|
|
198
|
+
if use_reservations else 0
|
|
199
|
+
)
|
|
200
|
+
memory_reserve = min(memory_reserve, raw_memory)
|
|
201
|
+
else:
|
|
202
|
+
raw_memory = int(_nd("memory_total"))
|
|
203
|
+
memory_reserve = 0
|
|
204
|
+
|
|
205
|
+
nodes.append(Node(
|
|
206
|
+
name=name,
|
|
207
|
+
cpu_total=int(_nd("cpu_total")),
|
|
208
|
+
memory_total=raw_memory,
|
|
209
|
+
memory_reserve=memory_reserve,
|
|
210
|
+
storage_free={"local": int(_nd("disk_free"))},
|
|
211
|
+
cpu_pressure=_nd("cpu_pressure_some_percent", 0.0),
|
|
212
|
+
memory_pressure=_nd("memory_pressure_some_percent", 0.0),
|
|
213
|
+
io_pressure=_nd("disk_pressure_some_percent", 0.0),
|
|
214
|
+
maintenance=bool(_nd("maintenance", False)),
|
|
215
|
+
))
|
|
216
|
+
|
|
217
|
+
# 3. Guests + constraint extraction ----------------------------------------
|
|
218
|
+
vms: list[VM] = []
|
|
219
|
+
affinity_groups: dict[tuple[str, str], list[str]] = defaultdict(list)
|
|
220
|
+
anti_affinity_groups: dict[tuple[str, str], list[str]] = defaultdict(list)
|
|
221
|
+
pin_rules: list[dict[str, Any]] = []
|
|
222
|
+
ignore_list: list[str] = []
|
|
223
|
+
|
|
224
|
+
pools_config = balancing_cfg.get("pools") or {}
|
|
225
|
+
|
|
226
|
+
for name, gd in proxlb_data.get("guests", {}).items():
|
|
227
|
+
|
|
228
|
+
def _gd(key: str, default: Any = 0, _gd: dict[str, Any] = gd) -> Any: # noqa: E731
|
|
229
|
+
return _gd.get(key, default)
|
|
230
|
+
|
|
231
|
+
vms.append(VM(
|
|
232
|
+
name=name,
|
|
233
|
+
node=_gd("node_current", ""),
|
|
234
|
+
cpu=int(_gd("cpu_total", 1)),
|
|
235
|
+
memory=int(_gd("memory_total")),
|
|
236
|
+
cpu_usage=float(_gd("cpu_used")),
|
|
237
|
+
cpu_pressure=_gd("cpu_pressure_some_percent", 0.0),
|
|
238
|
+
memory_pressure=_gd("memory_pressure_some_percent", 0.0),
|
|
239
|
+
io_pressure=_gd("disk_pressure_some_percent", 0.0),
|
|
240
|
+
disks={"local": int(_gd("disk_used"))},
|
|
241
|
+
priority=int(_gd("priority", 2)),
|
|
242
|
+
vm_type=str(_gd("type", "vm")),
|
|
243
|
+
))
|
|
244
|
+
|
|
245
|
+
# A. Native Proxmox HA rules
|
|
246
|
+
for rule in _gd("ha_rules", []):
|
|
247
|
+
r_id = str(rule.get("rule") if isinstance(rule, dict) else getattr(rule, "rule", ""))
|
|
248
|
+
r_type = rule.get("type") if isinstance(rule, dict) else str(getattr(rule, "type", ""))
|
|
249
|
+
if r_type == "affinity":
|
|
250
|
+
affinity_groups[(r_id, "pve")].append(name)
|
|
251
|
+
elif r_type == "anti-affinity":
|
|
252
|
+
anti_affinity_groups[(r_id, "pve")].append(name)
|
|
253
|
+
|
|
254
|
+
# B. ProxLB tags (plb_affinity_*, plb_anti_affinity_*)
|
|
255
|
+
for tag in _gd("tags", []):
|
|
256
|
+
if tag.startswith("plb_affinity"):
|
|
257
|
+
affinity_groups[(tag, "plb")].append(name)
|
|
258
|
+
elif tag.startswith("plb_anti_affinity"):
|
|
259
|
+
anti_affinity_groups[(tag, "plb")].append(name)
|
|
260
|
+
|
|
261
|
+
# C. Resource pool rules
|
|
262
|
+
for pool_name in _gd("pools", []):
|
|
263
|
+
pool_cfg = pools_config.get(pool_name) if isinstance(pools_config, dict) else None
|
|
264
|
+
if pool_cfg:
|
|
265
|
+
p_type = pool_cfg.get("type") if isinstance(pool_cfg, dict) else str(getattr(pool_cfg, "type", ""))
|
|
266
|
+
if p_type == "affinity":
|
|
267
|
+
affinity_groups[(pool_name, "plb")].append(name)
|
|
268
|
+
elif p_type == "anti-affinity":
|
|
269
|
+
anti_affinity_groups[(pool_name, "plb")].append(name)
|
|
270
|
+
|
|
271
|
+
# D. Node pinning (tags, pools, HA restricted nodes)
|
|
272
|
+
node_rels = _gd("node_relationships", [])
|
|
273
|
+
if node_rels:
|
|
274
|
+
origins: list[dict[str, Any]] = []
|
|
275
|
+
for tag in _gd("tags", []):
|
|
276
|
+
if tag.startswith("plb_pin"):
|
|
277
|
+
origins.append({"origin": "tag", "source": tag})
|
|
278
|
+
for pool in _gd("pools", []):
|
|
279
|
+
pool_cfg = pools_config.get(pool) if isinstance(pools_config, dict) else None
|
|
280
|
+
if pool_cfg:
|
|
281
|
+
has_pin = pool_cfg.get("pin") if isinstance(pool_cfg, dict) else getattr(pool_cfg, "pin", None)
|
|
282
|
+
if has_pin:
|
|
283
|
+
origins.append({"origin": "pool", "source": pool})
|
|
284
|
+
for rule in _gd("ha_rules", []):
|
|
285
|
+
r_type = rule.get("type") if isinstance(rule, dict) else str(getattr(rule, "type", ""))
|
|
286
|
+
r_nodes = rule.get("nodes") if isinstance(rule, dict) else getattr(rule, "nodes", [])
|
|
287
|
+
r_id = str(rule.get("rule") if isinstance(rule, dict) else getattr(rule, "rule", ""))
|
|
288
|
+
if r_type == "affinity" and r_nodes:
|
|
289
|
+
origins.append({"origin": "pve", "source": r_id})
|
|
290
|
+
pin_rules.append({
|
|
291
|
+
"vm": name,
|
|
292
|
+
"nodes": node_rels,
|
|
293
|
+
"origins": origins,
|
|
294
|
+
})
|
|
295
|
+
|
|
296
|
+
# E. Active-mode feedback: pin VMs that failed to migrate previously
|
|
297
|
+
if pin_vms and name in pin_vms:
|
|
298
|
+
pin_rules.append({
|
|
299
|
+
"vm": name,
|
|
300
|
+
"nodes": [_gd("node_current", "")],
|
|
301
|
+
"origins": [{"origin": "solver", "source": "migration_failed"}],
|
|
302
|
+
})
|
|
303
|
+
|
|
304
|
+
# F. Ignore flag
|
|
305
|
+
if _gd("ignore", False):
|
|
306
|
+
ignore_list.append(name)
|
|
307
|
+
|
|
308
|
+
# 4. Assemble Constraints --------------------------------------------------
|
|
309
|
+
constraints = Constraints(
|
|
310
|
+
affinity=[
|
|
311
|
+
{"name": k[0], "origin": k[1], "vms": v, "hard": True}
|
|
312
|
+
for k, v in affinity_groups.items() if len(v) > 1
|
|
313
|
+
],
|
|
314
|
+
anti_affinity=[
|
|
315
|
+
{"name": k[0], "origin": k[1], "vms": v, "hard": True}
|
|
316
|
+
for k, v in anti_affinity_groups.items() if len(v) > 1
|
|
317
|
+
],
|
|
318
|
+
pin=pin_rules,
|
|
319
|
+
ignore=ignore_list,
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
cluster_name = meta.get("cluster_name", "Live Cluster") if isinstance(meta, dict) else "Live Cluster"
|
|
323
|
+
|
|
324
|
+
return Cluster(
|
|
325
|
+
name=cluster_name,
|
|
326
|
+
description="Auto-generated from ProxLB live data",
|
|
327
|
+
balancing=balancing,
|
|
328
|
+
nodes=nodes,
|
|
329
|
+
vms=vms,
|
|
330
|
+
constraints=constraints,
|
|
331
|
+
expect=Expect(feasible=True),
|
|
332
|
+
)
|
proxlb_solver/cli.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""CLI entry point for generating reports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .loader import load_scenario
|
|
10
|
+
from .planner import plan_migrations
|
|
11
|
+
from .reporter import (
|
|
12
|
+
_check_expectations,
|
|
13
|
+
print_report,
|
|
14
|
+
write_html_report,
|
|
15
|
+
write_junit_xml,
|
|
16
|
+
write_markdown_report,
|
|
17
|
+
)
|
|
18
|
+
from .solver import solve, solve_reachable
|
|
19
|
+
|
|
20
|
+
# Development fallback: scenarios/ next to the repo root.
|
|
21
|
+
# This path does not exist in an installed package; users must pass --scenarios.
|
|
22
|
+
_DEV_SCENARIOS_DIR = Path(__file__).parent.parent / "scenarios"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def collect_scenarios(base: Path) -> list[Path]:
|
|
26
|
+
return sorted(base.rglob("*.yaml"))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main() -> None:
|
|
30
|
+
parser = argparse.ArgumentParser(
|
|
31
|
+
description="ProxLB CP-SAT Solver — run scenarios and generate reports"
|
|
32
|
+
)
|
|
33
|
+
parser.add_argument(
|
|
34
|
+
"--scenarios", type=Path, default=None,
|
|
35
|
+
help=(
|
|
36
|
+
"Directory containing YAML scenario files. "
|
|
37
|
+
"Defaults to scenarios/ relative to the source tree when running "
|
|
38
|
+
"from a checkout; required when running from an installed package."
|
|
39
|
+
),
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"--markdown", type=Path, default=None,
|
|
43
|
+
help="Write Markdown report to this path",
|
|
44
|
+
)
|
|
45
|
+
parser.add_argument(
|
|
46
|
+
"--html", type=Path, default=None,
|
|
47
|
+
help="Write HTML report to this path",
|
|
48
|
+
)
|
|
49
|
+
parser.add_argument(
|
|
50
|
+
"--junit", type=Path, default=None,
|
|
51
|
+
help="Write JUnit XML report to this path",
|
|
52
|
+
)
|
|
53
|
+
parser.add_argument(
|
|
54
|
+
"--quiet", action="store_true",
|
|
55
|
+
help="Suppress terminal output",
|
|
56
|
+
)
|
|
57
|
+
args = parser.parse_args()
|
|
58
|
+
|
|
59
|
+
if args.scenarios is None:
|
|
60
|
+
if _DEV_SCENARIOS_DIR.is_dir():
|
|
61
|
+
args.scenarios = _DEV_SCENARIOS_DIR
|
|
62
|
+
else:
|
|
63
|
+
parser.error(
|
|
64
|
+
"--scenarios is required when running from an installed package.\n"
|
|
65
|
+
"Example: proxlb-solver --scenarios /path/to/scenarios"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
scenario_files = collect_scenarios(args.scenarios)
|
|
69
|
+
if not scenario_files:
|
|
70
|
+
print("No scenarios found in %s" % args.scenarios, file=sys.stderr)
|
|
71
|
+
sys.exit(1)
|
|
72
|
+
|
|
73
|
+
results = []
|
|
74
|
+
results_junit = []
|
|
75
|
+
migration_plans = {}
|
|
76
|
+
|
|
77
|
+
for path in scenario_files:
|
|
78
|
+
rel = str(path.relative_to(args.scenarios))
|
|
79
|
+
cluster = load_scenario(path)
|
|
80
|
+
|
|
81
|
+
# Use reachable solver (feedback loop)
|
|
82
|
+
solution, mig_plan = solve_reachable(cluster, quiet=args.quiet)
|
|
83
|
+
|
|
84
|
+
migration_plans[rel] = mig_plan
|
|
85
|
+
|
|
86
|
+
if not args.quiet:
|
|
87
|
+
print_report(cluster, solution)
|
|
88
|
+
|
|
89
|
+
results.append((rel, cluster, solution))
|
|
90
|
+
|
|
91
|
+
# Evaluate errors for JUnit
|
|
92
|
+
checks = _check_expectations(cluster, solution, mig_plan)
|
|
93
|
+
errors = [
|
|
94
|
+
"%s: %s" % (name, detail)
|
|
95
|
+
for name, _, passed, detail in checks if not passed
|
|
96
|
+
]
|
|
97
|
+
results_junit.append((rel, cluster, solution, errors))
|
|
98
|
+
|
|
99
|
+
if args.markdown:
|
|
100
|
+
write_markdown_report(results, args.markdown, migration_plans)
|
|
101
|
+
if not args.quiet:
|
|
102
|
+
print("\nMarkdown report written to %s" % args.markdown)
|
|
103
|
+
|
|
104
|
+
if args.html:
|
|
105
|
+
write_html_report(results, args.html, migration_plans)
|
|
106
|
+
if not args.quiet:
|
|
107
|
+
print("HTML report written to %s" % args.html)
|
|
108
|
+
|
|
109
|
+
if args.junit:
|
|
110
|
+
write_junit_xml(results_junit, args.junit)
|
|
111
|
+
if not args.quiet:
|
|
112
|
+
print("JUnit XML written to %s" % args.junit)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
if __name__ == "__main__":
|
|
116
|
+
main()
|
proxlb_solver/loader.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""YAML scenario loader."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from collections import defaultdict
|
|
7
|
+
|
|
8
|
+
import yaml
|
|
9
|
+
|
|
10
|
+
from .models import Balancing, Cluster, Constraints, Expect, Node, VM
|
|
11
|
+
|
|
12
|
+
_GB = 1024 * 1024 * 1024
|
|
13
|
+
_MB = 1024 * 1024
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _gb_to_bytes(value: float) -> int:
|
|
17
|
+
return int(value * _GB)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def load_scenario(path: str | Path) -> Cluster:
|
|
21
|
+
"""Load a YAML scenario file and return a Cluster."""
|
|
22
|
+
path = Path(path)
|
|
23
|
+
with open(path) as f:
|
|
24
|
+
data = yaml.safe_load(f)
|
|
25
|
+
|
|
26
|
+
balancing_data = data.get("balancing", {})
|
|
27
|
+
balanciness = balancing_data.get("balanciness", 3)
|
|
28
|
+
if not (1 <= balanciness <= 5):
|
|
29
|
+
raise ValueError(
|
|
30
|
+
f"balanciness must be 1–5, got {balanciness}"
|
|
31
|
+
)
|
|
32
|
+
balancing = Balancing(
|
|
33
|
+
method=balancing_data.get("method", "memory"),
|
|
34
|
+
mode=balancing_data.get("mode", "used"),
|
|
35
|
+
balanciness=balanciness,
|
|
36
|
+
cpu_overcommit=balancing_data.get("cpu_overcommit", 2.0),
|
|
37
|
+
memory_threshold=balancing_data.get("memory_threshold"),
|
|
38
|
+
cpu_threshold=balancing_data.get("cpu_threshold"),
|
|
39
|
+
disk_threshold=balancing_data.get("disk_threshold"),
|
|
40
|
+
w_balance=balancing_data.get("w_balance"),
|
|
41
|
+
w_stickiness=balancing_data.get("w_stickiness"),
|
|
42
|
+
w_cpu_usage=balancing_data.get("w_cpu_usage", 1),
|
|
43
|
+
w_cpu_psi=balancing_data.get("w_cpu_psi", 1),
|
|
44
|
+
w_mem_usage=balancing_data.get("w_mem_usage", 1),
|
|
45
|
+
w_mem_psi=balancing_data.get("w_mem_psi", 1),
|
|
46
|
+
w_io_usage=balancing_data.get("w_io_usage", 1),
|
|
47
|
+
w_io_psi=balancing_data.get("w_io_psi", 1),
|
|
48
|
+
w_global_mem=balancing_data.get("w_global_mem", 10),
|
|
49
|
+
w_global_cpu=balancing_data.get("w_global_cpu", 10),
|
|
50
|
+
w_global_io=balancing_data.get("w_global_io", 1),
|
|
51
|
+
max_parallel_migrations=balancing_data.get("max_parallel_migrations"),
|
|
52
|
+
max_node_inflow=balancing_data.get("max_node_inflow", 1),
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
nodes = []
|
|
56
|
+
for name, nd in data.get("nodes", {}).items():
|
|
57
|
+
storage_free = {
|
|
58
|
+
sname: _gb_to_bytes(sval)
|
|
59
|
+
for sname, sval in nd.get("storage_free", {}).items()
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
reserve_data = nd.get("reserve", {})
|
|
63
|
+
storage_reserve = {
|
|
64
|
+
sname: _gb_to_bytes(sval)
|
|
65
|
+
for sname, sval in reserve_data.get("storage_gb", {}).items()
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
nodes.append(Node(
|
|
69
|
+
name=name,
|
|
70
|
+
cpu_total=nd["cpu_total"],
|
|
71
|
+
memory_total=_gb_to_bytes(nd["memory_total_gb"]),
|
|
72
|
+
storage_free=storage_free,
|
|
73
|
+
cpu_reserve=reserve_data.get("cpu", 0),
|
|
74
|
+
memory_reserve=_gb_to_bytes(reserve_data.get("memory_gb", 0)),
|
|
75
|
+
storage_reserve=storage_reserve,
|
|
76
|
+
cpu_pressure=nd.get("cpu_pressure", 0.0),
|
|
77
|
+
memory_pressure=nd.get("memory_pressure", 0.0),
|
|
78
|
+
io_pressure=nd.get("io_pressure", 0.0),
|
|
79
|
+
maintenance=nd.get("maintenance", False),
|
|
80
|
+
))
|
|
81
|
+
|
|
82
|
+
vms = []
|
|
83
|
+
tag_affinity = defaultdict(list)
|
|
84
|
+
tag_anti_affinity = defaultdict(list)
|
|
85
|
+
tag_pin = []
|
|
86
|
+
|
|
87
|
+
for name, vd in data.get("vms", {}).items():
|
|
88
|
+
disks = {
|
|
89
|
+
sname: _gb_to_bytes(sval)
|
|
90
|
+
for sname, sval in vd.get("disks", {}).items()
|
|
91
|
+
}
|
|
92
|
+
vms.append(VM(
|
|
93
|
+
name=name,
|
|
94
|
+
node=vd["node"],
|
|
95
|
+
cpu=vd["cpu"],
|
|
96
|
+
memory=_gb_to_bytes(vd["memory_gb"]),
|
|
97
|
+
cpu_usage=vd.get("cpu_usage", float(vd["cpu"])),
|
|
98
|
+
cpu_pressure=vd.get("cpu_pressure", 0.0),
|
|
99
|
+
memory_pressure=vd.get("memory_pressure", 0.0),
|
|
100
|
+
io_pressure=vd.get("io_pressure", 0.0),
|
|
101
|
+
disks=disks,
|
|
102
|
+
priority=vd.get("priority", 2),
|
|
103
|
+
vm_type=vd.get("type", "vm"),
|
|
104
|
+
))
|
|
105
|
+
|
|
106
|
+
# Parse tags for implicit constraints
|
|
107
|
+
for tag in vd.get("tags", []):
|
|
108
|
+
if tag.startswith("plb_affinity_"):
|
|
109
|
+
tag_affinity[tag].append(name)
|
|
110
|
+
elif tag.startswith("plb_anti_affinity_"):
|
|
111
|
+
tag_anti_affinity[tag].append(name)
|
|
112
|
+
elif tag.startswith("plb_pin_"):
|
|
113
|
+
target_node = tag[len("plb_pin_"):]
|
|
114
|
+
tag_pin.append({"vm": name, "nodes": [target_node], "origins": [{"origin": "tag", "source": tag}]})
|
|
115
|
+
|
|
116
|
+
cd = data.get("constraints", {})
|
|
117
|
+
ignore_raw = cd.get("ignore", [])
|
|
118
|
+
ignore_list = []
|
|
119
|
+
for entry in ignore_raw:
|
|
120
|
+
if isinstance(entry, dict):
|
|
121
|
+
ignore_list.append(entry["vm"])
|
|
122
|
+
else:
|
|
123
|
+
ignore_list.append(entry)
|
|
124
|
+
|
|
125
|
+
affinity = [{**r, "hard": r.get("hard", True), "origin": r.get("origin", "plb")} for r in cd.get("affinity", [])]
|
|
126
|
+
for tag, t_vms in tag_affinity.items():
|
|
127
|
+
if len(t_vms) > 1:
|
|
128
|
+
affinity.append({"name": tag, "vms": t_vms, "hard": True, "origin": "plb"})
|
|
129
|
+
|
|
130
|
+
anti_affinity = [{**r, "hard": r.get("hard", True), "origin": r.get("origin", "plb")} for r in cd.get("anti_affinity", [])]
|
|
131
|
+
for tag, t_vms in tag_anti_affinity.items():
|
|
132
|
+
if len(t_vms) > 1:
|
|
133
|
+
anti_affinity.append({"name": tag, "vms": t_vms, "hard": True, "origin": "plb"})
|
|
134
|
+
|
|
135
|
+
pin = cd.get("pin", []) + tag_pin
|
|
136
|
+
|
|
137
|
+
constraints = Constraints(
|
|
138
|
+
affinity=affinity,
|
|
139
|
+
anti_affinity=anti_affinity,
|
|
140
|
+
pin=pin,
|
|
141
|
+
ignore=ignore_list,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
ed = data.get("expect", {})
|
|
145
|
+
placements_raw = ed.get("placements", {})
|
|
146
|
+
placements = {k: str(v) for k, v in placements_raw.items()}
|
|
147
|
+
|
|
148
|
+
expect = Expect(
|
|
149
|
+
feasible=ed.get("feasible", True),
|
|
150
|
+
constraints_satisfied=ed.get("constraints_satisfied", True),
|
|
151
|
+
spread_improved=ed.get("spread_improved"),
|
|
152
|
+
max_migrations=ed.get("max_migrations"),
|
|
153
|
+
placements=placements,
|
|
154
|
+
node_empty=ed.get("node_empty"),
|
|
155
|
+
path_feasible=ed.get("path_feasible"),
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
evacuate_node = data.get("evacuate_node")
|
|
159
|
+
|
|
160
|
+
# Validate references
|
|
161
|
+
node_names = {n.name for n in nodes}
|
|
162
|
+
for vm in vms:
|
|
163
|
+
if vm.node not in node_names:
|
|
164
|
+
raise ValueError(
|
|
165
|
+
f"VM '{vm.name}' references unknown node '{vm.node}'"
|
|
166
|
+
)
|
|
167
|
+
if evacuate_node and evacuate_node not in node_names:
|
|
168
|
+
raise ValueError(
|
|
169
|
+
f"evacuate_node '{evacuate_node}' is not a known node"
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
return Cluster(
|
|
173
|
+
name=data.get("name", path.stem),
|
|
174
|
+
description=data.get("description", ""),
|
|
175
|
+
balancing=balancing,
|
|
176
|
+
nodes=nodes,
|
|
177
|
+
vms=vms,
|
|
178
|
+
constraints=constraints,
|
|
179
|
+
expect=expect,
|
|
180
|
+
evacuate_node=evacuate_node,
|
|
181
|
+
)
|