proxlb 1.1.12.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etc/proxlb/proxlb_example.yaml +90 -0
- etc/systemd/system/proxlb.service +13 -0
- proxlb/__init__.py +0 -0
- proxlb/main.py +120 -0
- proxlb/models/__init__.py +0 -0
- proxlb/models/balancing.py +270 -0
- proxlb/models/calculations.py +889 -0
- proxlb/models/features.py +124 -0
- proxlb/models/groups.py +124 -0
- proxlb/models/guests.py +215 -0
- proxlb/models/ha_rules.py +126 -0
- proxlb/models/nodes.py +312 -0
- proxlb/models/pools.py +148 -0
- proxlb/models/tags.py +283 -0
- proxlb/utils/__init__.py +0 -0
- proxlb/utils/cli_parser.py +90 -0
- proxlb/utils/config_parser.py +96 -0
- proxlb/utils/helper.py +337 -0
- proxlb/utils/logger.py +146 -0
- proxlb/utils/proxmox_api.py +440 -0
- proxlb/utils/version.py +7 -0
- proxlb-1.1.12.dev4.dist-info/METADATA +26 -0
- proxlb-1.1.12.dev4.dist-info/RECORD +26 -0
- proxlb-1.1.12.dev4.dist-info/WHEEL +5 -0
- proxlb-1.1.12.dev4.dist-info/licenses/LICENSE +674 -0
- proxlb-1.1.12.dev4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,889 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The Calculations class is responsible for handling the balancing of virtual machines (VMs)
|
|
3
|
+
and containers (CTs) across all available nodes in a Proxmox cluster. It provides methods
|
|
4
|
+
to calculate the optimal distribution of VMs and CTs based on the provided data.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
|
9
|
+
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
|
10
|
+
__license__ = "GPL-3.0"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
import sys
|
|
14
|
+
from typing import Dict, Any
|
|
15
|
+
from utils.logger import SystemdLogger
|
|
16
|
+
|
|
17
|
+
logger = SystemdLogger()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Calculations:
|
|
21
|
+
"""
|
|
22
|
+
The calculation class is responsible for handling the balancing of virtual machines (VMs)
|
|
23
|
+
and containers (CTs) across all available nodes in a Proxmox cluster. It provides methods
|
|
24
|
+
to calculate the optimal distribution of VMs and CTs based on the provided data.
|
|
25
|
+
|
|
26
|
+
Methods:
|
|
27
|
+
__init__(proxlb_data: Dict[str, Any]):
|
|
28
|
+
Initializes the Calculation class with the provided ProxLB data.
|
|
29
|
+
|
|
30
|
+
set_node_assignments(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
|
31
|
+
Sets the assigned resources of the nodes based on the current assigned
|
|
32
|
+
guest resources by their created groups as an initial base.
|
|
33
|
+
|
|
34
|
+
get_balanciness(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
|
35
|
+
Gets the balanciness for further actions where the highest and lowest
|
|
36
|
+
usage or assignments of Proxmox nodes are compared.
|
|
37
|
+
|
|
38
|
+
get_most_free_node(proxlb_data: Dict[str, Any], return_node: bool = False) -> Dict[str, Any]:
|
|
39
|
+
Gets the name of the Proxmox node in the cluster with the most free resources based on
|
|
40
|
+
the user-defined method (e.g., memory) and mode (e.g., used).
|
|
41
|
+
|
|
42
|
+
relocate_guests_on_maintenance_nodes(proxlb_data: Dict[str, Any]):
|
|
43
|
+
Relocates guests that are currently on nodes marked for maintenance to
|
|
44
|
+
nodes with the most available resources.
|
|
45
|
+
|
|
46
|
+
relocate_guests(proxlb_data: Dict[str, Any]):
|
|
47
|
+
Relocates guests within the provided data structure to ensure affinity groups are
|
|
48
|
+
placed on nodes with the most free resources.
|
|
49
|
+
|
|
50
|
+
val_anti_affinity(proxlb_data: Dict[str, Any], guest_name: str):
|
|
51
|
+
Validates and assigns nodes to guests based on anti-affinity rules.
|
|
52
|
+
|
|
53
|
+
update_node_resources(proxlb_data):
|
|
54
|
+
Updates the resource allocation and usage statistics for nodes when a guest
|
|
55
|
+
is moved from one node to another.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def __init__(self, proxlb_data: Dict[str, Any]):
|
|
59
|
+
"""
|
|
60
|
+
Initializes the Calculation class with the provided ProxLB data.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
proxlb_data (Dict[str, Any]): The data required for balancing VMs and CTs.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def set_node_assignments(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
|
68
|
+
"""
|
|
69
|
+
Set the assigned resources of the nodes based on the current assigned
|
|
70
|
+
guest resources by their created groups as an initial base.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
proxlb_data (Dict[str, Any]): The data holding all current statistics.
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
Dict[str, Any]: Updated ProxLB data of nodes section with updated node assigned values.
|
|
77
|
+
"""
|
|
78
|
+
logger.debug("Starting: set_node_assignments.")
|
|
79
|
+
for group_name, group_meta in proxlb_data["groups"]["affinity"].items():
|
|
80
|
+
|
|
81
|
+
for guest_name in group_meta["guests"]:
|
|
82
|
+
guest_node_current = proxlb_data["guests"][guest_name]["node_current"]
|
|
83
|
+
# Update resource assignments
|
|
84
|
+
# Update assigned values for the current node
|
|
85
|
+
logger.debug(f"set_node_assignment of guest {guest_name} on node {guest_node_current} with cpu_total: {proxlb_data['guests'][guest_name]['cpu_total']}, memory_total: {proxlb_data['guests'][guest_name]['memory_total']}, disk_total: {proxlb_data['guests'][guest_name]['disk_total']}.")
|
|
86
|
+
proxlb_data["nodes"][guest_node_current]["cpu_assigned"] += proxlb_data["guests"][guest_name]["cpu_total"]
|
|
87
|
+
proxlb_data["nodes"][guest_node_current]["memory_assigned"] += proxlb_data["guests"][guest_name]["memory_total"]
|
|
88
|
+
proxlb_data["nodes"][guest_node_current]["disk_assigned"] += proxlb_data["guests"][guest_name]["disk_total"]
|
|
89
|
+
# Update assigned percentage values for the current node
|
|
90
|
+
proxlb_data["nodes"][guest_node_current]["cpu_assigned_percent"] = proxlb_data["nodes"][guest_node_current]["cpu_assigned"] / proxlb_data["nodes"][guest_node_current]["cpu_total"] * 100
|
|
91
|
+
proxlb_data["nodes"][guest_node_current]["memory_assigned_percent"] = proxlb_data["nodes"][guest_node_current]["memory_assigned"] / proxlb_data["nodes"][guest_node_current]["memory_total"] * 100
|
|
92
|
+
proxlb_data["nodes"][guest_node_current]["disk_assigned_percent"] = proxlb_data["nodes"][guest_node_current]["disk_assigned"] / proxlb_data["nodes"][guest_node_current]["disk_total"] * 100
|
|
93
|
+
|
|
94
|
+
logger.debug("Finished: set_node_assignments.")
|
|
95
|
+
|
|
96
|
+
def set_node_hot(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
|
97
|
+
"""
|
|
98
|
+
Evaluates node 'full' pressure metrics for memory, cpu, and io
|
|
99
|
+
against defined thresholds and sets <metric>_pressure_hot = True
|
|
100
|
+
when a node is considered HOT.
|
|
101
|
+
|
|
102
|
+
Returns the modified proxlb_data dict.
|
|
103
|
+
"""
|
|
104
|
+
logger.debug("Starting: set_node_hot.")
|
|
105
|
+
balancing_cfg = proxlb_data.get("meta", {}).get("balancing", {})
|
|
106
|
+
thresholds = balancing_cfg.get("psi_thresholds", balancing_cfg.get("psi", {}).get("nodes", {}))
|
|
107
|
+
nodes = proxlb_data.get("nodes", {})
|
|
108
|
+
|
|
109
|
+
for node_name, node in nodes.items():
|
|
110
|
+
|
|
111
|
+
if node.get("maintenance"):
|
|
112
|
+
continue
|
|
113
|
+
|
|
114
|
+
if node.get("ignore"):
|
|
115
|
+
continue
|
|
116
|
+
|
|
117
|
+
# PSI metrics are only availavble on Proxmox VE 9.0 and higher.
|
|
118
|
+
if proxlb_data["meta"]["balancing"].get("mode", "used") == "psi":
|
|
119
|
+
|
|
120
|
+
if tuple(map(int, proxlb_data["nodes"][node["name"]]["pve_version"].split('.'))) < tuple(map(int, "9.0".split('.'))):
|
|
121
|
+
logger.critical(f"Proxmox node {node['name']} runs Proxmox VE version {proxlb_data['nodes'][node['name']]['pve_version']}."
|
|
122
|
+
" PSI metrics require Proxmox VE 9.0 or higher. Balancing deactivated!")
|
|
123
|
+
|
|
124
|
+
for metric, threshold in thresholds.items():
|
|
125
|
+
pressure_full = node.get(f"{metric}_pressure_full_percent", 0.0)
|
|
126
|
+
pressure_some = node.get(f"{metric}_pressure_some_percent", 0.0)
|
|
127
|
+
pressure_spikes = node.get(f"{metric}_pressure_full_spikes_percent", 0.0)
|
|
128
|
+
is_hot = (pressure_full >= threshold["pressure_full"] and pressure_some >= threshold["pressure_some"]) or (pressure_spikes >= threshold["pressure_spikes"])
|
|
129
|
+
|
|
130
|
+
if is_hot:
|
|
131
|
+
logger.debug(f"Set node {node['name']} as hot based on {metric} pressure metrics.")
|
|
132
|
+
proxlb_data["nodes"][node["name"]][f"{metric}_pressure_hot"] = True
|
|
133
|
+
proxlb_data["nodes"][node["name"]][f"pressure_hot"] = True
|
|
134
|
+
else:
|
|
135
|
+
logger.debug(f"Node {node['name']} is not hot based on {metric} pressure metrics.")
|
|
136
|
+
|
|
137
|
+
logger.debug("Finished: set_node_hot.")
|
|
138
|
+
return proxlb_data
|
|
139
|
+
|
|
140
|
+
def set_guest_hot(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
|
141
|
+
"""
|
|
142
|
+
Evaluates guest 'full' pressure metrics for memory, cpu, and io
|
|
143
|
+
against defined thresholds and sets <metric>_pressure_hot = True
|
|
144
|
+
when a guest is considered HOT.
|
|
145
|
+
|
|
146
|
+
Returns the modified proxlb_data dict.
|
|
147
|
+
"""
|
|
148
|
+
logger.debug("Starting: set_guest_hot.")
|
|
149
|
+
balancing_cfg = proxlb_data.get("meta", {}).get("balancing", {})
|
|
150
|
+
thresholds = balancing_cfg.get("psi_thresholds", balancing_cfg.get("psi", {}).get("guests", {}))
|
|
151
|
+
guests = proxlb_data.get("guests", {})
|
|
152
|
+
|
|
153
|
+
for guest_name, guest in guests.items():
|
|
154
|
+
if guest.get("ignore"):
|
|
155
|
+
continue
|
|
156
|
+
|
|
157
|
+
for metric, threshold in thresholds.items():
|
|
158
|
+
pressure_full = guest.get(f"{metric}_pressure_full_percent", 0.0)
|
|
159
|
+
pressure_some = guest.get(f"{metric}_pressure_some_percent", 0.0)
|
|
160
|
+
pressure_spikes = guest.get(f"{metric}_pressure_full_spikes_percent", 0.0)
|
|
161
|
+
is_hot = (pressure_full >= threshold["pressure_full"] and pressure_some >= threshold["pressure_some"]) or (pressure_spikes >= threshold["pressure_spikes"])
|
|
162
|
+
|
|
163
|
+
if is_hot:
|
|
164
|
+
logger.debug(f"Set guest {guest['name']} as hot based on {metric} pressure metrics.")
|
|
165
|
+
proxlb_data["guests"][guest["name"]][f"{metric}_pressure_hot"] = True
|
|
166
|
+
proxlb_data["guests"][guest["name"]][f"pressure_hot"] = True
|
|
167
|
+
else:
|
|
168
|
+
logger.debug(f"guest {guest['name']} is not hot based on {metric} pressure metrics.")
|
|
169
|
+
|
|
170
|
+
logger.debug("Finished: set_guest_hot.")
|
|
171
|
+
return proxlb_data
|
|
172
|
+
|
|
173
|
+
@staticmethod
|
|
174
|
+
def get_balanciness(proxlb_data: Dict[str, Any]) -> Dict[str, Any]:
|
|
175
|
+
"""
|
|
176
|
+
Get the blanaciness for further actions where the highest and lowest
|
|
177
|
+
usage or assignments of Proxmox nodes are compared. Based on the users
|
|
178
|
+
provided balanciness delta the balancing will be performed.
|
|
179
|
+
|
|
180
|
+
Args:
|
|
181
|
+
proxlb_data (Dict[str, Any]): The data holding all content of all objects.
|
|
182
|
+
Returns:
|
|
183
|
+
Dict[str, Any]: Updated meta data section of the balanciness action defined
|
|
184
|
+
as a bool.
|
|
185
|
+
"""
|
|
186
|
+
logger.debug("Starting: get_balanciness.")
|
|
187
|
+
proxlb_data["meta"]["balancing"]["balance"] = False
|
|
188
|
+
|
|
189
|
+
if len(proxlb_data["groups"]) > 0:
|
|
190
|
+
method = proxlb_data["meta"]["balancing"].get("method", "memory")
|
|
191
|
+
mode = proxlb_data["meta"]["balancing"].get("mode", "used")
|
|
192
|
+
balanciness = proxlb_data["meta"]["balancing"].get("balanciness", 10)
|
|
193
|
+
|
|
194
|
+
if mode == "assigned":
|
|
195
|
+
method_value = [node_meta[f"{method}_{mode}_percent"] for node_meta in proxlb_data["nodes"].values()]
|
|
196
|
+
|
|
197
|
+
if proxlb_data["meta"]["balancing"].get(f"{method}_threshold", None):
|
|
198
|
+
threshold = proxlb_data["meta"]["balancing"].get(f"{method}_threshold")
|
|
199
|
+
# if the key isn't present, we assume 100% usage
|
|
200
|
+
highest_usage_node = max(proxlb_data["nodes"].values(), key=lambda x: x.get(f"{method}_{mode}_percent", 100))
|
|
201
|
+
highest_node_value = highest_usage_node[f"{method}_{mode}_percent"]
|
|
202
|
+
|
|
203
|
+
if highest_node_value >= threshold:
|
|
204
|
+
logger.debug(f"Guest balancing is required. Highest {method} usage node {highest_usage_node['name']} is above the defined threshold of {threshold}% with a value of {highest_node_value}%.")
|
|
205
|
+
proxlb_data["meta"]["balancing"]["balance"] = True
|
|
206
|
+
else:
|
|
207
|
+
logger.debug(f"Guest balancing is ok. Highest {method} usage node {highest_usage_node['name']} is below the defined threshold of {threshold}% with a value of {highest_node_value}%.")
|
|
208
|
+
proxlb_data["meta"]["balancing"]["balance"] = False
|
|
209
|
+
|
|
210
|
+
else:
|
|
211
|
+
logger.debug(f"No {method} threshold defined for balancing. Skipping threshold check.")
|
|
212
|
+
|
|
213
|
+
elif mode == "used":
|
|
214
|
+
method_value = [node_meta[f"{method}_{mode}_percent"] for node_meta in proxlb_data["nodes"].values()]
|
|
215
|
+
|
|
216
|
+
if proxlb_data["meta"]["balancing"].get(f"{method}_threshold", None):
|
|
217
|
+
threshold = proxlb_data["meta"]["balancing"].get(f"{method}_threshold")
|
|
218
|
+
# if the key isn't present, we assume 100% usage
|
|
219
|
+
highest_usage_node = max(proxlb_data["nodes"].values(), key=lambda x: x.get(f"{method}_{mode}_percent", 100))
|
|
220
|
+
highest_node_value = highest_usage_node[f"{method}_{mode}_percent"]
|
|
221
|
+
|
|
222
|
+
if highest_node_value >= threshold:
|
|
223
|
+
logger.debug(f"Guest balancing is required. Highest {method} usage node {highest_usage_node['name']} is above the defined threshold of {threshold}% with a value of {highest_node_value}%.")
|
|
224
|
+
proxlb_data["meta"]["balancing"]["balance"] = True
|
|
225
|
+
else:
|
|
226
|
+
logger.debug(f"Guest balancing is ok. Highest {method} usage node {highest_usage_node['name']} is below the defined threshold of {threshold}% with a value of {highest_node_value}%.")
|
|
227
|
+
proxlb_data["meta"]["balancing"]["balance"] = False
|
|
228
|
+
|
|
229
|
+
else:
|
|
230
|
+
logger.debug(f"No {method} threshold defined for balancing. Skipping threshold check.")
|
|
231
|
+
|
|
232
|
+
elif mode == "psi":
|
|
233
|
+
method_value = [node_meta[f"{method}_pressure_full_spikes_percent"] for node_meta in proxlb_data["nodes"].values()]
|
|
234
|
+
any_node_hot = any(node.get(f"{method}_pressure_hot", False) for node in proxlb_data["nodes"].values())
|
|
235
|
+
any_guest_hot = any(node.get(f"{method}_pressure_hot", False) for node in proxlb_data["guests"].values())
|
|
236
|
+
|
|
237
|
+
if any_node_hot:
|
|
238
|
+
logger.debug(f"Guest balancing is required. A node is marked as HOT based on {method} pressure metrics.")
|
|
239
|
+
proxlb_data["meta"]["balancing"]["balance"] = True
|
|
240
|
+
else:
|
|
241
|
+
logger.debug(f"Guest balancing is ok. No node is marked as HOT based on {method} pressure metrics.")
|
|
242
|
+
|
|
243
|
+
if any_guest_hot:
|
|
244
|
+
logger.debug(f"Guest balancing is required. A guest is marked as HOT based on {method} pressure metrics.")
|
|
245
|
+
proxlb_data["meta"]["balancing"]["balance"] = True
|
|
246
|
+
else:
|
|
247
|
+
logger.debug(f"Guest balancing is ok. No guest is marked as HOT based on {method} pressure metrics.")
|
|
248
|
+
|
|
249
|
+
return proxlb_data
|
|
250
|
+
|
|
251
|
+
else:
|
|
252
|
+
logger.critical(f"Unknown balancing mode: {mode} provided. Cannot get balanciness.")
|
|
253
|
+
sys.exit(1)
|
|
254
|
+
|
|
255
|
+
method_value_highest = max(method_value)
|
|
256
|
+
method_value_lowest = min(method_value)
|
|
257
|
+
|
|
258
|
+
if method_value_highest - method_value_lowest > balanciness:
|
|
259
|
+
proxlb_data["meta"]["balancing"]["balance"] = True
|
|
260
|
+
logger.debug(f"Guest balancing is required. Highest value: {method_value_highest}, lowest value: {method_value_lowest} balanced by {method} and {mode}.")
|
|
261
|
+
else:
|
|
262
|
+
logger.debug(f"Guest balancing is ok. Highest value: {method_value_highest}, lowest value: {method_value_lowest} balanced by {method} and {mode}.")
|
|
263
|
+
|
|
264
|
+
else:
|
|
265
|
+
logger.warning("No guests for balancing found.")
|
|
266
|
+
|
|
267
|
+
logger.debug("Finished: get_balanciness.")
|
|
268
|
+
|
|
269
|
+
@staticmethod
|
|
270
|
+
def get_most_free_node(proxlb_data: Dict[str, Any], return_node: bool = False, guest_node_relation_list: list = []) -> Dict[str, Any]:
|
|
271
|
+
"""
|
|
272
|
+
Get the name of the Proxmox node in the cluster with the most free resources based on
|
|
273
|
+
the user defined method (e.g.: memory) and mode (e.g.: used).
|
|
274
|
+
|
|
275
|
+
Args:
|
|
276
|
+
proxlb_data (Dict[str, Any]): The data holding all content of all objects.
|
|
277
|
+
return_node (bool): The indicator to simply return the best node for further
|
|
278
|
+
assignments.
|
|
279
|
+
guest_node_relation_list (list): A list of nodes that have a tag on the given
|
|
280
|
+
guest relationship for pinning.
|
|
281
|
+
|
|
282
|
+
Returns:
|
|
283
|
+
Dict[str, Any]: Updated meta data section of the node with the most free resources that should
|
|
284
|
+
be used for the next balancing action.
|
|
285
|
+
"""
|
|
286
|
+
logger.debug("Starting: get_most_free_node.")
|
|
287
|
+
proxlb_data["meta"]["balancing"]["balance_next_node"] = None
|
|
288
|
+
|
|
289
|
+
# insure that we don't error out if "nodes" key is missing
|
|
290
|
+
nodes_dict = proxlb_data.get("nodes", {})
|
|
291
|
+
|
|
292
|
+
# Filter and include nodes that given by a relationship between guest and node. This is only
|
|
293
|
+
# used if the guest has a relationship to a node defined by "pin" tags.
|
|
294
|
+
if len(guest_node_relation_list) > 0:
|
|
295
|
+
filtered_nodes = [
|
|
296
|
+
node
|
|
297
|
+
for node in nodes_dict.values()
|
|
298
|
+
if node["name"] in guest_node_relation_list and not node["maintenance"]
|
|
299
|
+
]
|
|
300
|
+
else:
|
|
301
|
+
# Filter and exclude nodes only that are in maintenance mode
|
|
302
|
+
filtered_nodes = [
|
|
303
|
+
node for node in nodes_dict.values() if not node["maintenance"]
|
|
304
|
+
]
|
|
305
|
+
|
|
306
|
+
if not filtered_nodes:
|
|
307
|
+
# log an error if filtered_nodes is empty
|
|
308
|
+
logger.critical("No possible target nodes found for balancing after applying maintenance and relationship filters.")
|
|
309
|
+
proxlb_data["meta"]["balancing"]["balance_reason"] = "resources"
|
|
310
|
+
proxlb_data["meta"]["balancing"]["balance_next_node"] = None
|
|
311
|
+
return None
|
|
312
|
+
|
|
313
|
+
# Filter by the defined methods and modes for balancing
|
|
314
|
+
method = proxlb_data["meta"]["balancing"].get("method", "memory")
|
|
315
|
+
mode = proxlb_data["meta"]["balancing"].get("mode", "used")
|
|
316
|
+
|
|
317
|
+
if mode == "assigned":
|
|
318
|
+
logger.debug(f"Get best node for balancing by assigned {method} resources.")
|
|
319
|
+
lowest_usage_node = min(
|
|
320
|
+
filtered_nodes, key=lambda x: x.get(f"{method}_{mode}_percent", 100)
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
elif mode == "used":
|
|
324
|
+
logger.debug(f"Get best node for balancing by used {method} resources.")
|
|
325
|
+
lowest_usage_node = min(
|
|
326
|
+
filtered_nodes, key=lambda x: x.get(f"{method}_{mode}_percent", 100)
|
|
327
|
+
)
|
|
328
|
+
elif mode == "psi":
|
|
329
|
+
logger.debug(
|
|
330
|
+
f"Get best node for balancing by pressure of {method} resources."
|
|
331
|
+
)
|
|
332
|
+
lowest_usage_node = min(
|
|
333
|
+
filtered_nodes,
|
|
334
|
+
key=lambda x: x.get(f"{method}_pressure_full_spikes_percent", 100),
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
else:
|
|
338
|
+
logger.critical(
|
|
339
|
+
f"Unknown balancing mode: {mode} provided. Cannot get best node."
|
|
340
|
+
)
|
|
341
|
+
sys.exit(1)
|
|
342
|
+
|
|
343
|
+
if not lowest_usage_node:
|
|
344
|
+
logger.critical("No suitable node found for balancing.")
|
|
345
|
+
proxlb_data["meta"]["balancing"]["balance_reason"] = "resources"
|
|
346
|
+
proxlb_data["meta"]["balancing"]["balance_next_node"] = None
|
|
347
|
+
else:
|
|
348
|
+
proxlb_data["meta"]["balancing"]["balance_reason"] = "resources"
|
|
349
|
+
proxlb_data["meta"]["balancing"]["balance_next_node"] = lowest_usage_node["name"]
|
|
350
|
+
logger.debug(f"Best node for balancing is: {lowest_usage_node['name']}.")
|
|
351
|
+
|
|
352
|
+
# If executed to simply get the best node for further usage, we return
|
|
353
|
+
# the best node on stdout and gracefully exit here
|
|
354
|
+
if return_node:
|
|
355
|
+
print(lowest_usage_node.get("name", None))
|
|
356
|
+
sys.exit(0)
|
|
357
|
+
|
|
358
|
+
logger.debug("Finished: get_most_free_node.")
|
|
359
|
+
return lowest_usage_node
|
|
360
|
+
|
|
361
|
+
@staticmethod
|
|
362
|
+
def relocate_guests_on_maintenance_nodes(proxlb_data: Dict[str, Any]):
|
|
363
|
+
"""
|
|
364
|
+
Relocates guests that are currently on nodes marked for maintenance to
|
|
365
|
+
nodes with the most available resources.
|
|
366
|
+
|
|
367
|
+
This function iterates over all guests on maintenance nodes and attempts
|
|
368
|
+
to relocate them to nodes with the most free resources that are not in
|
|
369
|
+
maintenance mode. It updates the node resources accordingly and logs
|
|
370
|
+
warnings if the balancing may not be perfect due to the maintenance
|
|
371
|
+
status of the original node.
|
|
372
|
+
|
|
373
|
+
Args:
|
|
374
|
+
proxlb_data (Dict[str, Any]): The data holding all content of all objects.
|
|
375
|
+
Returns:
|
|
376
|
+
None
|
|
377
|
+
"""
|
|
378
|
+
logger.debug("Starting: relocate_guests_on_maintenance_nodes.")
|
|
379
|
+
proxlb_data["meta"]["balancing"]["balance_next_guest"] = ""
|
|
380
|
+
|
|
381
|
+
for guest_name in proxlb_data["groups"]["maintenance"]:
|
|
382
|
+
# Update the node with the most free nodes which is
|
|
383
|
+
# not in a maintenance
|
|
384
|
+
proxlb_data["meta"]["balancing"]["balance_next_guest"] = guest_name
|
|
385
|
+
if Calculations.get_most_free_node(proxlb_data):
|
|
386
|
+
Calculations.update_node_resources(proxlb_data)
|
|
387
|
+
else:
|
|
388
|
+
logger.warning(f"Warning: Could not find a suitable node to relocate guest {guest_name} from a maintenance node.")
|
|
389
|
+
logger.warning(f"Warning: Balancing may not be perfect because guest {guest_name} was located on a node which is in maintenance mode.")
|
|
390
|
+
|
|
391
|
+
logger.debug("Finished: relocate_guests_on_maintenance_nodes.")
|
|
392
|
+
|
|
393
|
+
@staticmethod
|
|
394
|
+
def relocate_guests(proxlb_data: Dict[str, Any]):
|
|
395
|
+
"""
|
|
396
|
+
Relocates guests within the provided data structure to ensure affinity groups are
|
|
397
|
+
placed on nodes with the most free resources.
|
|
398
|
+
|
|
399
|
+
This function iterates over each affinity group in the provided data, identifies
|
|
400
|
+
the node with the most free resources, and migrates all guests within the group
|
|
401
|
+
to that node. It updates the node resources accordingly.
|
|
402
|
+
|
|
403
|
+
Args:
|
|
404
|
+
proxlb_data (Dict[str, Any]): The data holding all content of all objects.
|
|
405
|
+
Returns:
|
|
406
|
+
None
|
|
407
|
+
"""
|
|
408
|
+
logger.debug("Starting: relocate_guests.")
|
|
409
|
+
|
|
410
|
+
# Balance only if it is required by:
|
|
411
|
+
# - balanciness
|
|
412
|
+
# - Affinity/Anti-Affinity rules
|
|
413
|
+
# - Pinning rules
|
|
414
|
+
if proxlb_data["meta"]["balancing"]["balance"] or proxlb_data["meta"]["balancing"].get("enforce_affinity", False) or proxlb_data["meta"]["balancing"].get("enforce_pinning", False):
|
|
415
|
+
|
|
416
|
+
if proxlb_data["meta"]["balancing"].get("balance", False):
|
|
417
|
+
logger.debug("Balancing of guests will be performed. Reason: balanciness")
|
|
418
|
+
|
|
419
|
+
if proxlb_data["meta"]["balancing"].get("enforce_affinity", False):
|
|
420
|
+
logger.debug("Balancing of guests will be performed. Reason: enforce affinity balancing")
|
|
421
|
+
|
|
422
|
+
if proxlb_data["meta"]["balancing"].get("enforce_pinning", False):
|
|
423
|
+
logger.debug("Balancing of guests will be performed. Reason: enforce pinning balancing")
|
|
424
|
+
|
|
425
|
+
# Sort guests by used memory
|
|
426
|
+
# Allows processing larger guests first or smaller guests first
|
|
427
|
+
larger_first = proxlb_data.get("meta", {}).get("balancing", {}).get("balance_larger_guests_first", False)
|
|
428
|
+
|
|
429
|
+
if larger_first:
|
|
430
|
+
logger.debug("Larger guests will be processed first. (Sorting descending by memory used)")
|
|
431
|
+
else:
|
|
432
|
+
logger.debug("Smaller guests will be processed first. (Sorting ascending by memory used)")
|
|
433
|
+
|
|
434
|
+
# Sort affinity groups by number of guests to avoid creating more migrations than needed
|
|
435
|
+
# because of affinity-groups and use afterwards memory for defining smaller/larger guests
|
|
436
|
+
sorted_guest_usage_groups = sorted(
|
|
437
|
+
proxlb_data["groups"]["affinity"],
|
|
438
|
+
key=lambda g: (
|
|
439
|
+
proxlb_data["groups"]["affinity"][g]["counter"],
|
|
440
|
+
-proxlb_data["groups"]["affinity"][g]["memory_used"]
|
|
441
|
+
if larger_first
|
|
442
|
+
else proxlb_data["groups"]["affinity"][g]["memory_used"],
|
|
443
|
+
)
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
# Iterate over all affinity groups
|
|
447
|
+
for group_name in sorted_guest_usage_groups:
|
|
448
|
+
|
|
449
|
+
# Validate balanciness again before processing each group
|
|
450
|
+
Calculations.get_balanciness(proxlb_data)
|
|
451
|
+
logger.debug(proxlb_data["meta"]["balancing"]["balance"])
|
|
452
|
+
|
|
453
|
+
if (not proxlb_data["meta"]["balancing"]["balance"]) and (not proxlb_data["meta"]["balancing"].get("enforce_affinity", False)) and (not proxlb_data["meta"]["balancing"].get("enforce_pinning", False)):
|
|
454
|
+
logger.debug("Skipping further guest relocations as balanciness is now ok.")
|
|
455
|
+
break
|
|
456
|
+
|
|
457
|
+
for guest_name in proxlb_data["groups"]["affinity"][group_name]["guests"]:
|
|
458
|
+
|
|
459
|
+
# Stop moving guests if the source node is no longer the most loaded
|
|
460
|
+
source_node = proxlb_data["guests"][guest_name]["node_current"]
|
|
461
|
+
method = proxlb_data["meta"]["balancing"].get("method", "memory")
|
|
462
|
+
mode = proxlb_data["meta"]["balancing"].get("mode", "used")
|
|
463
|
+
highest_node = max(proxlb_data["nodes"].values(), key=lambda n: n[f"{method}_used_percent"])
|
|
464
|
+
|
|
465
|
+
if highest_node["name"] != source_node:
|
|
466
|
+
logger.debug(f"Stopping relocation for guest {guest_name}: source node {source_node} is no longer the most loaded node.")
|
|
467
|
+
break
|
|
468
|
+
|
|
469
|
+
if not Calculations.validate_node_resources(proxlb_data, guest_name):
|
|
470
|
+
if proxlb_data['meta']['balancing'].get('balance_next_node', None) is None:
|
|
471
|
+
logger.warning(f"Skipping relocation of guest {guest_name} due to no target node defined. This might affect affinity group {group_name}.")
|
|
472
|
+
else:
|
|
473
|
+
logger.warning(f"Skipping relocation of guest {guest_name} due to insufficient resources on target node {proxlb_data['meta']['balancing']['balance_next_node']}. This might affect affinity group {group_name}.")
|
|
474
|
+
continue
|
|
475
|
+
|
|
476
|
+
if mode == 'psi':
|
|
477
|
+
logger.debug(f"Evaluating guest relocation based on {mode} mode.")
|
|
478
|
+
method = proxlb_data["meta"]["balancing"].get("method", "memory")
|
|
479
|
+
processed_guests_psi = proxlb_data["meta"]["balancing"].setdefault("processed_guests_psi", [])
|
|
480
|
+
unprocessed_guests_psi = [guest for guest in proxlb_data["guests"].values() if guest["name"] not in processed_guests_psi]
|
|
481
|
+
|
|
482
|
+
# Filter by the defined methods and modes for balancing
|
|
483
|
+
highest_usage_guest = max(unprocessed_guests_psi, key=lambda x: x[f"{method}_pressure_full_spikes_percent"])
|
|
484
|
+
|
|
485
|
+
# Append guest to the psi based processed list of guests
|
|
486
|
+
if highest_usage_guest["name"] == guest_name and guest_name not in proxlb_data["meta"]["balancing"]["processed_guests_psi"]:
|
|
487
|
+
proxlb_data["meta"]["balancing"]["processed_guests_psi"].append(guest_name)
|
|
488
|
+
proxlb_data["meta"]["balancing"]["balance_next_guest"] = guest_name
|
|
489
|
+
|
|
490
|
+
else:
|
|
491
|
+
logger.debug(f"Evaluating guest relocation based on {mode} mode.")
|
|
492
|
+
proxlb_data["meta"]["balancing"]["balance_next_guest"] = guest_name
|
|
493
|
+
|
|
494
|
+
Calculations.val_anti_affinity(proxlb_data, guest_name)
|
|
495
|
+
Calculations.val_node_relationships(proxlb_data, guest_name)
|
|
496
|
+
Calculations.update_node_resources(proxlb_data)
|
|
497
|
+
|
|
498
|
+
logger.debug("Finished: relocate_guests.")
|
|
499
|
+
|
|
500
|
+
@staticmethod
|
|
501
|
+
def val_anti_affinity(proxlb_data: Dict[str, Any], guest_name: str):
|
|
502
|
+
"""
|
|
503
|
+
Validates and assigns nodes to guests based on anti-affinity rules.
|
|
504
|
+
|
|
505
|
+
This function iterates over all defined anti-affinity groups in the provided
|
|
506
|
+
`proxlb_data` and checks if the specified `guest_name` is included in any of
|
|
507
|
+
these groups. If the guest is included and has not been processed yet, it
|
|
508
|
+
attempts to assign an unused and non-maintenance node to the guest, ensuring
|
|
509
|
+
that the anti-affinity rules are respected.
|
|
510
|
+
|
|
511
|
+
Parameters:
|
|
512
|
+
proxlb_data (Dict[str, Any]): The data holding all content of all objects.
|
|
513
|
+
guest_name (str): The name of the guest to be validated and assigned a node.
|
|
514
|
+
|
|
515
|
+
Returns:
|
|
516
|
+
None
|
|
517
|
+
"""
|
|
518
|
+
logger.debug("Starting: val_anti_affinity.")
|
|
519
|
+
# Start by iterating over all defined anti-affinity groups
|
|
520
|
+
for group_name in proxlb_data["groups"]["anti_affinity"].keys():
|
|
521
|
+
|
|
522
|
+
# Validate if the provided guest is included in the anti-affinity group
|
|
523
|
+
if guest_name in proxlb_data["groups"]["anti_affinity"][group_name]['guests'] and not proxlb_data["guests"][guest_name]["processed"]:
|
|
524
|
+
logger.debug(f"Anti-Affinity: Guest: {guest_name} is included in anti-affinity group: {group_name}.")
|
|
525
|
+
|
|
526
|
+
# Check if the group has only one member. If so skip new guest node assignment.
|
|
527
|
+
if proxlb_data["groups"]["anti_affinity"][group_name]["counter"] > 1:
|
|
528
|
+
logger.debug(f"Anti-Affinity: Group has more than 1 member.")
|
|
529
|
+
# Iterate over all available nodes
|
|
530
|
+
for node_name in proxlb_data["nodes"].keys():
|
|
531
|
+
|
|
532
|
+
# Only select node if it was not used before and is not in a
|
|
533
|
+
# maintenance mode. Afterwards, add it to the list of already
|
|
534
|
+
# used nodes for the current anti-affinity group
|
|
535
|
+
if node_name not in proxlb_data["groups"]["anti_affinity"][group_name]["used_nodes"]:
|
|
536
|
+
|
|
537
|
+
if not proxlb_data["nodes"][node_name]["maintenance"]:
|
|
538
|
+
# If the node has not been used yet, we assign this node to the guest
|
|
539
|
+
proxlb_data["meta"]["balancing"]["balance_next_node"] = node_name
|
|
540
|
+
proxlb_data["groups"]["anti_affinity"][group_name]["used_nodes"].append(node_name)
|
|
541
|
+
logger.debug(f"Node: {node_name} marked as used for anti-affinity group: {group_name} with guest {guest_name}")
|
|
542
|
+
break
|
|
543
|
+
|
|
544
|
+
else:
|
|
545
|
+
logger.critical(f"Node: {node_name} already got used for anti-affinity group:: {group_name}. (Tried for guest: {guest_name})")
|
|
546
|
+
else:
|
|
547
|
+
logger.debug(f"Anti-Affinity: Group has less than 2 members. Skipping node calculation for the group.")
|
|
548
|
+
|
|
549
|
+
else:
|
|
550
|
+
logger.debug(f"Guest: {guest_name} is not included in anti-affinity group: {group_name}. Skipping.")
|
|
551
|
+
|
|
552
|
+
logger.debug("Finished: val_anti_affinity.")
|
|
553
|
+
|
|
554
|
+
@staticmethod
|
|
555
|
+
def val_node_relationships(proxlb_data: Dict[str, Any], guest_name: str):
|
|
556
|
+
"""
|
|
557
|
+
Validates and assigns guests to nodes based on defined relationships based on tags.
|
|
558
|
+
|
|
559
|
+
Parameters:
|
|
560
|
+
proxlb_data (Dict[str, Any]): The data holding all content of all objects.
|
|
561
|
+
guest_name (str): The name of the guest to be validated and assigned a node.
|
|
562
|
+
|
|
563
|
+
Returns:
|
|
564
|
+
None
|
|
565
|
+
"""
|
|
566
|
+
logger.debug("Starting: val_node_relationships.")
|
|
567
|
+
proxlb_data["guests"][guest_name]["processed"] = True
|
|
568
|
+
|
|
569
|
+
if len(proxlb_data["guests"][guest_name]["node_relationships"]) > 0:
|
|
570
|
+
logger.debug(f"Guest '{guest_name}' has relationships defined to node(s): {','.join(proxlb_data['guests'][guest_name]['node_relationships'])}. Pinning to node.")
|
|
571
|
+
|
|
572
|
+
# Get the list of nodes that are defined as relationship for the guest
|
|
573
|
+
guest_node_relation_list = proxlb_data["guests"][guest_name]["node_relationships"]
|
|
574
|
+
|
|
575
|
+
# Validate if strict relationships are defined. If not, we prefer
|
|
576
|
+
# the most free node in addition to the relationship list.
|
|
577
|
+
if proxlb_data["guests"][guest_name]["node_relationships_strict"]:
|
|
578
|
+
logger.debug(f"Guest '{guest_name}' has strict node relationships defined. Only nodes in the relationship list will be considered for pinning.")
|
|
579
|
+
else:
|
|
580
|
+
logger.debug(f"Guest '{guest_name}' has non-strict node relationships defined. Prefering nodes in the relationship list for pinning.")
|
|
581
|
+
Calculations.get_most_free_node(proxlb_data)
|
|
582
|
+
most_free_node = proxlb_data["meta"]["balancing"]["balance_next_node"]
|
|
583
|
+
if most_free_node:
|
|
584
|
+
guest_node_relation_list.append(most_free_node)
|
|
585
|
+
|
|
586
|
+
# Get the most free node from the relationship list, or the most free node overall
|
|
587
|
+
Calculations.get_most_free_node(proxlb_data, False, guest_node_relation_list)
|
|
588
|
+
|
|
589
|
+
# Validate if the specified node name is really part of the cluster
|
|
590
|
+
if proxlb_data["meta"]["balancing"]["balance_next_node"] in proxlb_data["nodes"].keys():
|
|
591
|
+
logger.debug(f"Guest '{guest_name}' has a specific relationship defined to node: {proxlb_data['meta']['balancing']['balance_next_node']} is a known hypervisor node in the cluster.")
|
|
592
|
+
else:
|
|
593
|
+
logger.warning(f"Guest '{guest_name}' has a specific relationship defined to node: {proxlb_data['meta']['balancing']['balance_next_node']} but this node name is not known in the cluster!")
|
|
594
|
+
|
|
595
|
+
else:
|
|
596
|
+
logger.debug(f"Guest '{guest_name}' does not have any specific node relationships.")
|
|
597
|
+
|
|
598
|
+
logger.debug("Finished: val_node_relationships.")
|
|
599
|
+
|
|
600
|
+
@staticmethod
|
|
601
|
+
def update_node_resources(proxlb_data: Dict[str, Any]):
|
|
602
|
+
"""
|
|
603
|
+
Updates the resource allocation and usage statistics for nodes when a guest
|
|
604
|
+
is moved from one node to another.
|
|
605
|
+
|
|
606
|
+
Parameters:
|
|
607
|
+
proxlb_data (dict): A dictionary containing information about the nodes and
|
|
608
|
+
guests, including their resource allocations and usage.
|
|
609
|
+
|
|
610
|
+
The function performs the following steps:
|
|
611
|
+
1. Retrieves the guest name, current node, and target node from the provided data.
|
|
612
|
+
2. Updates the resource allocations and usage statistics for the target node by
|
|
613
|
+
adding the resources of the moved guest.
|
|
614
|
+
3. Updates the resource allocations and usage statistics for the current node by
|
|
615
|
+
subtracting the resources of the moved guest.
|
|
616
|
+
4. Logs the start and end of the resource update process, as well as the movement
|
|
617
|
+
of the guest from the current node to the target node.
|
|
618
|
+
"""
|
|
619
|
+
logger.debug("Starting: update_node_resources.")
|
|
620
|
+
guest_name = proxlb_data["meta"]["balancing"].get("balance_next_guest", None)
|
|
621
|
+
|
|
622
|
+
# with fix #24 an empty guest name will be None, but maybe there are still cases with empty strings
|
|
623
|
+
if guest_name is None or guest_name == "":
|
|
624
|
+
logger.debug("No guest defined to update node resources for.")
|
|
625
|
+
return
|
|
626
|
+
|
|
627
|
+
node_current = proxlb_data["guests"][guest_name]["node_current"]
|
|
628
|
+
node_target = proxlb_data["meta"]["balancing"].get("balance_next_node", None)
|
|
629
|
+
|
|
630
|
+
if node_target is None:
|
|
631
|
+
logger.debug(f"No target node defined to update node resources for guest {guest_name}.")
|
|
632
|
+
return
|
|
633
|
+
|
|
634
|
+
# Update resources for the target node by the moved guest resources
|
|
635
|
+
# Add assigned resources to the target node
|
|
636
|
+
proxlb_data["nodes"][node_target]["cpu_assigned"] += proxlb_data["guests"][guest_name]["cpu_total"]
|
|
637
|
+
proxlb_data["nodes"][node_target]["memory_assigned"] += proxlb_data["guests"][guest_name]["memory_total"]
|
|
638
|
+
proxlb_data["nodes"][node_target]["disk_assigned"] += proxlb_data["guests"][guest_name]["disk_total"]
|
|
639
|
+
# Update the assigned percentages of assigned resources for the target node
|
|
640
|
+
proxlb_data["nodes"][node_target]["cpu_assigned_percent"] = proxlb_data["nodes"][node_target]["cpu_assigned"] / proxlb_data["nodes"][node_target]["cpu_total"] * 100
|
|
641
|
+
proxlb_data["nodes"][node_target]["memory_assigned_percent"] = proxlb_data["nodes"][node_target]["memory_assigned"] / proxlb_data["nodes"][node_target]["memory_total"] * 100
|
|
642
|
+
proxlb_data["nodes"][node_target]["disk_assigned_percent"] = proxlb_data["nodes"][node_target]["disk_assigned"] / proxlb_data["nodes"][node_target]["disk_total"] * 100
|
|
643
|
+
# Add used resources to the target node
|
|
644
|
+
proxlb_data["nodes"][node_target]["cpu_used"] += proxlb_data["guests"][guest_name]["cpu_used"]
|
|
645
|
+
proxlb_data["nodes"][node_target]["memory_used"] += proxlb_data["guests"][guest_name]["memory_used"]
|
|
646
|
+
proxlb_data["nodes"][node_target]["disk_used"] += proxlb_data["guests"][guest_name]["disk_used"]
|
|
647
|
+
# Update the used percentages of usage resources for the target node
|
|
648
|
+
proxlb_data["nodes"][node_target]["cpu_used_percent"] = proxlb_data["nodes"][node_target]["cpu_used"] / proxlb_data["nodes"][node_target]["cpu_total"] * 100
|
|
649
|
+
proxlb_data["nodes"][node_target]["memory_used_percent"] = proxlb_data["nodes"][node_target]["memory_used"] / proxlb_data["nodes"][node_target]["memory_total"] * 100
|
|
650
|
+
proxlb_data["nodes"][node_target]["disk_used_percent"] = proxlb_data["nodes"][node_target]["disk_used"] / proxlb_data["nodes"][node_target]["disk_total"] * 100
|
|
651
|
+
|
|
652
|
+
# Update resources for the current node by the moved guest resources
|
|
653
|
+
# Add assigned resources to the target node
|
|
654
|
+
proxlb_data["nodes"][node_current]["cpu_assigned"] -= proxlb_data["guests"][guest_name]["cpu_total"]
|
|
655
|
+
proxlb_data["nodes"][node_current]["memory_assigned"] -= proxlb_data["guests"][guest_name]["memory_total"]
|
|
656
|
+
proxlb_data["nodes"][node_current]["disk_assigned"] -= proxlb_data["guests"][guest_name]["disk_total"]
|
|
657
|
+
# Update the assigned percentages of assigned resources for the target node
|
|
658
|
+
proxlb_data["nodes"][node_current]["cpu_assigned_percent"] = proxlb_data["nodes"][node_current]["cpu_assigned"] / proxlb_data["nodes"][node_current]["cpu_total"] * 100
|
|
659
|
+
proxlb_data["nodes"][node_current]["memory_assigned_percent"] = proxlb_data["nodes"][node_current]["memory_assigned"] / proxlb_data["nodes"][node_current]["memory_total"] * 100
|
|
660
|
+
proxlb_data["nodes"][node_current]["disk_assigned_percent"] = proxlb_data["nodes"][node_current]["disk_assigned"] / proxlb_data["nodes"][node_current]["disk_total"] * 100
|
|
661
|
+
# Add used resources to the target node
|
|
662
|
+
proxlb_data["nodes"][node_current]["cpu_used"] -= proxlb_data["guests"][guest_name]["cpu_used"]
|
|
663
|
+
proxlb_data["nodes"][node_current]["memory_used"] -= proxlb_data["guests"][guest_name]["memory_used"]
|
|
664
|
+
proxlb_data["nodes"][node_current]["disk_used"] -= proxlb_data["guests"][guest_name]["disk_used"]
|
|
665
|
+
# Update the used percentages of usage resources for the target node
|
|
666
|
+
proxlb_data["nodes"][node_current]["cpu_used_percent"] = proxlb_data["nodes"][node_current]["cpu_used"] / proxlb_data["nodes"][node_current]["cpu_total"] * 100
|
|
667
|
+
proxlb_data["nodes"][node_current]["memory_used_percent"] = proxlb_data["nodes"][node_current]["memory_used"] / proxlb_data["nodes"][node_current]["memory_total"] * 100
|
|
668
|
+
proxlb_data["nodes"][node_current]["disk_used_percent"] = proxlb_data["nodes"][node_current]["disk_used"] / proxlb_data["nodes"][node_current]["disk_total"] * 100
|
|
669
|
+
|
|
670
|
+
# Assign guest to the new target node
|
|
671
|
+
if not proxlb_data["guests"][guest_name]["ignore"]:
|
|
672
|
+
proxlb_data["guests"][guest_name]["node_target"] = node_target
|
|
673
|
+
logger.debug(f"Set guest {guest_name} from node {node_current} to node {node_target}.")
|
|
674
|
+
else:
|
|
675
|
+
logger.debug(f"Guest {guest_name} is marked as ignored. Skipping target node assignment.")
|
|
676
|
+
|
|
677
|
+
Calculations.recalc_node_statistics(proxlb_data, node_target)
|
|
678
|
+
Calculations.recalc_node_statistics(proxlb_data, node_current)
|
|
679
|
+
|
|
680
|
+
logger.debug("Finished: update_node_resources.")
|
|
681
|
+
|
|
682
|
+
def validate_affinity_map(proxlb_data: Dict[str, Any]):
|
|
683
|
+
"""
|
|
684
|
+
Validates the affinity and anti-affinity constraints for all guests in the ProxLB data structure.
|
|
685
|
+
|
|
686
|
+
This function iterates through each guest and checks both affinity and anti-affinity rules.
|
|
687
|
+
If any guest violates these constraints, it sets the enforce_affinity flag to trigger rebalancing
|
|
688
|
+
and skips further validation for efficiency.
|
|
689
|
+
|
|
690
|
+
Args:
|
|
691
|
+
proxlb_data (Dict[str, Any]): A dictionary containing ProxLB configuration with the following structure:
|
|
692
|
+
- "guests" (list): List of guest identifiers to validate
|
|
693
|
+
- "meta" (dict): Metadata dictionary containing:
|
|
694
|
+
- "balancing" (dict): Balancing configuration with "enforce_affinity" flag
|
|
695
|
+
|
|
696
|
+
Returns:
|
|
697
|
+
None: Modifies proxlb_data in-place by updating the "enforce_affinity" flag in meta.balancing
|
|
698
|
+
|
|
699
|
+
Raises:
|
|
700
|
+
None: Function handles validation gracefully and logs outcomes
|
|
701
|
+
"""
|
|
702
|
+
logger.debug("Starting: validate_current_affinity.")
|
|
703
|
+
balancing_ok = True
|
|
704
|
+
|
|
705
|
+
for guest in proxlb_data["guests"]:
|
|
706
|
+
|
|
707
|
+
# We do not need to validate anymore if rebalancing is required
|
|
708
|
+
if balancing_ok is False:
|
|
709
|
+
proxlb_data["meta"]["balancing"]["enforce_affinity"] = True
|
|
710
|
+
logger.debug(f"Rebalancing based on affinity/anti-affinity map is required. Skipping further validation...")
|
|
711
|
+
break
|
|
712
|
+
|
|
713
|
+
balancing_state_affinity = Calculations.validate_current_affinity(proxlb_data, guest)
|
|
714
|
+
balancing_state_anti_affinity = Calculations.validate_current_anti_affinity(proxlb_data, guest)
|
|
715
|
+
logger.debug(f"Affinity for guest {guest} is {'valid' if balancing_state_affinity else 'NOT valid'}")
|
|
716
|
+
logger.debug(f"Anti-affinity for guest {guest} is {'valid' if balancing_state_anti_affinity else 'NOT valid'}")
|
|
717
|
+
|
|
718
|
+
balancing_ok = balancing_state_affinity and balancing_state_anti_affinity
|
|
719
|
+
|
|
720
|
+
if balancing_ok:
|
|
721
|
+
logger.debug(f"Rebalancing based on affinity/anti-affinity map is not required.")
|
|
722
|
+
proxlb_data["meta"]["balancing"]["enforce_affinity"] = False
|
|
723
|
+
|
|
724
|
+
logger.debug("Finished: validate_current_affinity.")
|
|
725
|
+
|
|
726
|
+
@staticmethod
|
|
727
|
+
def get_guest_node(proxlb_data: Dict[str, Any], guest_name: str) -> str:
|
|
728
|
+
"""
|
|
729
|
+
Return a currently assoicated PVE node where the guest is running on.
|
|
730
|
+
|
|
731
|
+
Args:
|
|
732
|
+
proxlb_data (Dict[str, Any]): A dictionary containing ProxLB configuration.
|
|
733
|
+
|
|
734
|
+
Returns:
|
|
735
|
+
node_name_current (str): The name of the current node where the guest runs on.
|
|
736
|
+
|
|
737
|
+
"""
|
|
738
|
+
return proxlb_data["guests"][guest_name]["node_current"]
|
|
739
|
+
|
|
740
|
+
@staticmethod
|
|
741
|
+
def validate_current_affinity(proxlb_data: Dict[str, Any], guest_name: str) -> bool:
|
|
742
|
+
"""
|
|
743
|
+
Validate that all guests in affinity groups containing the specified guest are on the same non-maintenance node.
|
|
744
|
+
|
|
745
|
+
This function checks affinity group constraints for a given guest. It ensures that:
|
|
746
|
+
1. All guests within an affinity group are located on the same physical node
|
|
747
|
+
2. The node hosting the affinity group is not in maintenance mode
|
|
748
|
+
|
|
749
|
+
Args:
|
|
750
|
+
proxlb_data (Dict[str, Any]): A dictionary containing the complete ProxLB state including:
|
|
751
|
+
- "groups": Dictionary with "affinity" key containing affinity group definitions
|
|
752
|
+
- "guests": Dictionary with guest information
|
|
753
|
+
- "nodes": Dictionary with node information including maintenance status
|
|
754
|
+
guest_name (str): The name of the guest to validate affinity for
|
|
755
|
+
|
|
756
|
+
Returns:
|
|
757
|
+
bool: True if all affinity groups containing the guest are valid (all members on same
|
|
758
|
+
non-maintenance node), False otherwise
|
|
759
|
+
"""
|
|
760
|
+
logger.debug("Starting: validate_current_affinity.")
|
|
761
|
+
for group_name, grp in proxlb_data["groups"]["affinity"].items():
|
|
762
|
+
if guest_name not in grp["guests"]:
|
|
763
|
+
continue
|
|
764
|
+
|
|
765
|
+
nodes = []
|
|
766
|
+
for group in grp["guests"]:
|
|
767
|
+
if group not in proxlb_data["guests"]:
|
|
768
|
+
continue
|
|
769
|
+
|
|
770
|
+
node = Calculations.get_guest_node(proxlb_data, group)
|
|
771
|
+
if proxlb_data["nodes"][node]["maintenance"]:
|
|
772
|
+
logger.debug(f"Group '{group_name}' invalid: node '{node}' in maintenance.")
|
|
773
|
+
return False
|
|
774
|
+
nodes.append(node)
|
|
775
|
+
|
|
776
|
+
if len(set(nodes)) != 1:
|
|
777
|
+
logger.debug(f"Group '{group_name}' invalid: guests spread across nodes {set(nodes)}.")
|
|
778
|
+
return False
|
|
779
|
+
|
|
780
|
+
return True
|
|
781
|
+
|
|
782
|
+
@staticmethod
|
|
783
|
+
def validate_current_anti_affinity(proxlb_data: Dict[str, Any], guest_name: str) -> bool:
|
|
784
|
+
"""
|
|
785
|
+
Validate that all guests in anti-affinity groups containing the specified guest are not on the same node.
|
|
786
|
+
|
|
787
|
+
This function checks anti-affinity group constraints for a given guest. It ensures that:
|
|
788
|
+
1. All guests within an anti-affinity group are located on the same physical node
|
|
789
|
+
2. The node hosting the anti-affinity group is not in maintenance mode
|
|
790
|
+
|
|
791
|
+
Args:
|
|
792
|
+
proxlb_data (Dict[str, Any]): A dictionary containing the complete ProxLB state including:
|
|
793
|
+
- "groups": Dictionary with "affinity" key containing affinity group definitions
|
|
794
|
+
- "guests": Dictionary with guest information
|
|
795
|
+
- "nodes": Dictionary with node information including maintenance status
|
|
796
|
+
guest_name (str): The name of the guest to validate affinity for
|
|
797
|
+
|
|
798
|
+
Returns:
|
|
799
|
+
bool: True if all anti-affinity groups containing the guest are valid (all members on different
|
|
800
|
+
non-maintenance node), False otherwise
|
|
801
|
+
"""
|
|
802
|
+
logger.debug("Starting: validate_current_anti_affinity.")
|
|
803
|
+
for group_name, grp in proxlb_data["groups"]["anti_affinity"].items():
|
|
804
|
+
if guest_name not in grp["guests"]:
|
|
805
|
+
continue
|
|
806
|
+
nodes = []
|
|
807
|
+
for group in grp["guests"]:
|
|
808
|
+
if group not in proxlb_data["guests"]:
|
|
809
|
+
continue
|
|
810
|
+
|
|
811
|
+
node = Calculations.get_guest_node(proxlb_data, group)
|
|
812
|
+
if proxlb_data["nodes"][node]["maintenance"]:
|
|
813
|
+
return False
|
|
814
|
+
nodes.append(node)
|
|
815
|
+
|
|
816
|
+
if len(nodes) != len(set(nodes)):
|
|
817
|
+
return False
|
|
818
|
+
|
|
819
|
+
return True
|
|
820
|
+
|
|
821
|
+
@staticmethod
|
|
822
|
+
def validate_node_resources(proxlb_data: Dict[str, Any], guest_name: str) -> bool:
|
|
823
|
+
"""
|
|
824
|
+
Validate that the target node has sufficient resources to host the specified guest.
|
|
825
|
+
|
|
826
|
+
This function checks if the target node, determined by the balancing logic,
|
|
827
|
+
has enough CPU, memory, and disk resources available to accommodate the guest.
|
|
828
|
+
|
|
829
|
+
Args:
|
|
830
|
+
proxlb_data (Dict[str, Any]): A dictionary containing the complete ProxLB state including:
|
|
831
|
+
- "nodes": Dictionary with node resource information
|
|
832
|
+
- "guests": Dictionary with guest resource requirements
|
|
833
|
+
- "meta": Dictionary with balancing information including target node
|
|
834
|
+
guest_name (str): The name of the guest to validate resources for
|
|
835
|
+
Returns:
|
|
836
|
+
bool: True if the target node has sufficient resources, False otherwise
|
|
837
|
+
or if there is no target node defined.
|
|
838
|
+
"""
|
|
839
|
+
logger.debug("Starting: validate_node_resources.")
|
|
840
|
+
node_target = proxlb_data["meta"]["balancing"]["balance_next_node"]
|
|
841
|
+
|
|
842
|
+
if node_target is None:
|
|
843
|
+
logger.debug(f"No target node defined to validate node resources for guest {guest_name}.")
|
|
844
|
+
return False
|
|
845
|
+
|
|
846
|
+
node_memory_free = proxlb_data["nodes"][node_target]["memory_free"]
|
|
847
|
+
node_cpu_free = proxlb_data["nodes"][node_target]["cpu_free"]
|
|
848
|
+
node_disk_free = proxlb_data["nodes"][node_target]["disk_free"]
|
|
849
|
+
|
|
850
|
+
guest_memory_required = proxlb_data["guests"][guest_name]["memory_used"]
|
|
851
|
+
guest_cpu_required = proxlb_data["guests"][guest_name]["cpu_used"]
|
|
852
|
+
guest_disk_required = proxlb_data["guests"][guest_name]["disk_used"]
|
|
853
|
+
|
|
854
|
+
if guest_memory_required < node_memory_free:
|
|
855
|
+
logger.debug(f"Node '{node_target}' has sufficient resources ({node_memory_free / (1024 ** 3):.2f} GB free) for guest '{guest_name}'.")
|
|
856
|
+
logger.debug("Finished: validate_node_resources.")
|
|
857
|
+
return True
|
|
858
|
+
else:
|
|
859
|
+
logger.debug(f"Node '{node_target}' lacks sufficient resources ({node_memory_free / (1024 ** 3):.2f} GB free) for guest '{guest_name}'.")
|
|
860
|
+
logger.debug("Finished: validate_node_resources.")
|
|
861
|
+
return False
|
|
862
|
+
|
|
863
|
+
@staticmethod
|
|
864
|
+
def recalc_node_statistics(proxlb_data: Dict[str, Any], node_name: str) -> None:
|
|
865
|
+
"""
|
|
866
|
+
Recalculates node statistics including free resources and usage percentages.
|
|
867
|
+
|
|
868
|
+
This function updates the computed statistics for a node based on its current
|
|
869
|
+
resource allocation and usage. It calculates free resources, usage percentages,
|
|
870
|
+
and assigned percentages for CPU, memory, and disk.
|
|
871
|
+
|
|
872
|
+
Args:
|
|
873
|
+
proxlb_data (Dict[str, Any]): A dictionary containing the complete ProxLB state including:
|
|
874
|
+
- "nodes": Dictionary with node resource information
|
|
875
|
+
node_name (str): The name of the node to recalculate statistics for
|
|
876
|
+
|
|
877
|
+
Returns:
|
|
878
|
+
None: Modifies proxlb_data in-place by updating node statistics
|
|
879
|
+
"""
|
|
880
|
+
n = proxlb_data["nodes"][node_name]
|
|
881
|
+
n["cpu_free"] = max(0, n["cpu_total"] - n["cpu_used"])
|
|
882
|
+
n["memory_free"] = max(0, n["memory_total"] - n["memory_used"])
|
|
883
|
+
n["disk_free"] = max(0, n["disk_total"] - n["disk_used"])
|
|
884
|
+
n["cpu_used_percent"] = (n["cpu_used"] / n["cpu_total"] * 100) if n["cpu_total"] else 0
|
|
885
|
+
n["memory_used_percent"] = (n["memory_used"] / n["memory_total"] * 100) if n["memory_total"] else 0
|
|
886
|
+
n["disk_used_percent"] = (n["disk_used"] / n["disk_total"] * 100) if n["disk_total"] else 0
|
|
887
|
+
n["cpu_assigned_percent"] = (n["cpu_assigned"] / n["cpu_total"] * 100) if n["cpu_total"] else 0
|
|
888
|
+
n["memory_assigned_percent"] = (n["memory_assigned"] / n["memory_total"] * 100) if n["memory_total"] else 0
|
|
889
|
+
n["disk_assigned_percent"] = (n["disk_assigned"] / n["disk_total"] * 100) if n["disk_total"] else 0
|