proxlb 1.1.12.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etc/proxlb/proxlb_example.yaml +90 -0
- etc/systemd/system/proxlb.service +13 -0
- proxlb/__init__.py +0 -0
- proxlb/main.py +120 -0
- proxlb/models/__init__.py +0 -0
- proxlb/models/balancing.py +270 -0
- proxlb/models/calculations.py +889 -0
- proxlb/models/features.py +124 -0
- proxlb/models/groups.py +124 -0
- proxlb/models/guests.py +215 -0
- proxlb/models/ha_rules.py +126 -0
- proxlb/models/nodes.py +312 -0
- proxlb/models/pools.py +148 -0
- proxlb/models/tags.py +283 -0
- proxlb/utils/__init__.py +0 -0
- proxlb/utils/cli_parser.py +90 -0
- proxlb/utils/config_parser.py +96 -0
- proxlb/utils/helper.py +337 -0
- proxlb/utils/logger.py +146 -0
- proxlb/utils/proxmox_api.py +440 -0
- proxlb/utils/version.py +7 -0
- proxlb-1.1.12.dev4.dist-info/METADATA +26 -0
- proxlb-1.1.12.dev4.dist-info/RECORD +26 -0
- proxlb-1.1.12.dev4.dist-info/WHEEL +5 -0
- proxlb-1.1.12.dev4.dist-info/licenses/LICENSE +674 -0
- proxlb-1.1.12.dev4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ProxLB Features module for validating and adjusting feature flags
|
|
3
|
+
based on Proxmox VE node versions and cluster compatibility.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
|
7
|
+
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
|
8
|
+
__license__ = "GPL-3.0"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
from typing import List
|
|
12
|
+
from typing import Dict, Any
|
|
13
|
+
from utils.logger import SystemdLogger
|
|
14
|
+
from packaging import version
|
|
15
|
+
|
|
16
|
+
logger = SystemdLogger()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Features:
|
|
20
|
+
"""
|
|
21
|
+
ProxLB Features module for validating and adjusting feature flags
|
|
22
|
+
based on Proxmox VE node versions and cluster compatibility.
|
|
23
|
+
|
|
24
|
+
Responsibilities:
|
|
25
|
+
- Validate and adjust feature flags based on Proxmox VE node versions.
|
|
26
|
+
|
|
27
|
+
Methods:
|
|
28
|
+
__init__():
|
|
29
|
+
No-op initializer.
|
|
30
|
+
|
|
31
|
+
validate_available_features(proxlb_data: dict) -> None:
|
|
32
|
+
Static method that inspects proxlb_data["nodes"] versions and disables
|
|
33
|
+
incompatible balancing features for Proxmox VE versions < 9.0.0.
|
|
34
|
+
This function mutates proxlb_data in place.
|
|
35
|
+
|
|
36
|
+
Notes:
|
|
37
|
+
- Expects proxlb_data to be a dict with "nodes" and "meta" keys.
|
|
38
|
+
"""
|
|
39
|
+
def __init__(self):
|
|
40
|
+
"""
|
|
41
|
+
Initializes the Features class.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
def validate_available_features(proxlb_data: any) -> None:
|
|
46
|
+
"""
|
|
47
|
+
Validate and adjust feature flags in the provided proxlb_data according to Proxmox VE versions.
|
|
48
|
+
|
|
49
|
+
This function inspects the cluster node versions in proxlb_data and disables features
|
|
50
|
+
that are incompatible with Proxmox VE versions older than 9.0.0. Concretely, if any node
|
|
51
|
+
reports a 'pve_version' lower than "9.0.0":
|
|
52
|
+
- If meta.balancing.with_conntrack_state is truthy, it is set to False and a warning is logged.
|
|
53
|
+
- If meta.balancing.mode equals "psi", meta.balancing.enable is set to False and a warning is logged.
|
|
54
|
+
|
|
55
|
+
proxlb_data (dict): Cluster data structure that must contain:
|
|
56
|
+
- "nodes": a mapping (e.g., dict) whose values are mappings containing a 'pve_version' string.
|
|
57
|
+
- "meta": a mapping that may contain a "balancing" mapping with keys:
|
|
58
|
+
- "with_conntrack_state" (bool, optional)
|
|
59
|
+
- "mode" (str, optional)
|
|
60
|
+
- "enable" (bool, optional)
|
|
61
|
+
|
|
62
|
+
None: The function mutates proxlb_data in place to disable incompatible features.
|
|
63
|
+
|
|
64
|
+
Side effects:
|
|
65
|
+
- Mutates proxlb_data["meta"]["balancing"] when incompatible features are detected.
|
|
66
|
+
- Emits debug and warning log messages.
|
|
67
|
+
|
|
68
|
+
Notes:
|
|
69
|
+
- Unexpected or missing keys/types in proxlb_data may raise KeyError or TypeError.
|
|
70
|
+
- Version comparison uses semantic version parsing; callers should provide versions as strings.
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
None
|
|
74
|
+
"""
|
|
75
|
+
logger.debug("Starting: validate_available_features.")
|
|
76
|
+
|
|
77
|
+
any_non_pve9_node = any(version.parse(n['pve_version']) < version.parse("9.0.0") for n in proxlb_data["nodes"].values())
|
|
78
|
+
if any_non_pve9_node:
|
|
79
|
+
|
|
80
|
+
with_conntrack_state = proxlb_data["meta"].get("balancing", {}).get("with_conntrack_state", False)
|
|
81
|
+
if with_conntrack_state:
|
|
82
|
+
logger.warning("Non Proxmox VE 9 systems detected: Deactivating migration option 'with-conntrack-state'!")
|
|
83
|
+
proxlb_data["meta"]["balancing"]["with_conntrack_state"] = False
|
|
84
|
+
|
|
85
|
+
psi_balancing = proxlb_data["meta"].get("balancing", {}).get("mode", None)
|
|
86
|
+
if psi_balancing == "psi":
|
|
87
|
+
logger.warning("Non Proxmox VE 9 systems detected: Deactivating balancing!")
|
|
88
|
+
proxlb_data["meta"]["balancing"]["enable"] = False
|
|
89
|
+
|
|
90
|
+
logger.debug("Finished: validate_available_features.")
|
|
91
|
+
|
|
92
|
+
@staticmethod
|
|
93
|
+
def validate_any_non_pve9_node(meta: any, nodes: any) -> dict:
|
|
94
|
+
"""
|
|
95
|
+
Validate if any node in the cluster is running Proxmox VE < 9.0.0 and update meta accordingly.
|
|
96
|
+
|
|
97
|
+
This function inspects the cluster node versions and sets a flag in meta indicating whether
|
|
98
|
+
any node is running a Proxmox VE version older than 9.0.0.
|
|
99
|
+
|
|
100
|
+
Args:
|
|
101
|
+
meta (dict): Metadata structure that will be updated with cluster version information.
|
|
102
|
+
nodes (dict): Cluster nodes mapping whose values contain 'pve_version' strings.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
dict: The updated meta dictionary with 'cluster_non_pve9' flag set to True or False.
|
|
106
|
+
|
|
107
|
+
Side effects:
|
|
108
|
+
- Mutates meta["meta"]["cluster_non_pve9"] based on node versions.
|
|
109
|
+
- Emits debug log messages.
|
|
110
|
+
|
|
111
|
+
Notes:
|
|
112
|
+
- Version comparison uses semantic version parsing; defaults to "0.0.0" if pve_version is missing.
|
|
113
|
+
"""
|
|
114
|
+
logger.debug("Starting: validate_any_non_pve9_node.")
|
|
115
|
+
any_non_pve9_node = any(version.parse(node.get("pve_version", "0.0.0")) < version.parse("9.0.0") for node in nodes.get("nodes", {}).values())
|
|
116
|
+
|
|
117
|
+
if any_non_pve9_node:
|
|
118
|
+
meta["meta"]["cluster_non_pve9"] = True
|
|
119
|
+
logger.debug("Finished: validate_any_non_pve9_node. Result: True")
|
|
120
|
+
else:
|
|
121
|
+
meta["meta"]["cluster_non_pve9"] = False
|
|
122
|
+
logger.debug("Finished: validate_any_non_pve9_node. Result: False")
|
|
123
|
+
|
|
124
|
+
return meta
|
proxlb/models/groups.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The Groups class is responsible for handling the correlations between the guests
|
|
3
|
+
and their groups, such as affinity and anti-affinity groups. It ensures proper balancing
|
|
4
|
+
by grouping guests and evaluating them for further balancing. The class provides methods
|
|
5
|
+
to initialize with ProxLB data and to generate groups based on guest and node data.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
|
9
|
+
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
|
10
|
+
__license__ = "GPL-3.0"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
from typing import Dict, Any
|
|
14
|
+
from utils.logger import SystemdLogger
|
|
15
|
+
from utils.helper import Helper
|
|
16
|
+
|
|
17
|
+
logger = SystemdLogger()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Groups:
|
|
21
|
+
"""
|
|
22
|
+
The groups class is responsible for handling the correlations between the guests
|
|
23
|
+
and their groups like affinity and anti-affinity groups. To ensure a proper balancing
|
|
24
|
+
guests will ge grouped and then evaluated for further balancing.
|
|
25
|
+
|
|
26
|
+
Methods:
|
|
27
|
+
__init__(proxlb_data: Dict[str, Any]):
|
|
28
|
+
Initializes the Groups class.
|
|
29
|
+
|
|
30
|
+
get_groups(guests: Dict[str, Any], nodes: Dict[str, Any]) -> Dict[str, Any]:
|
|
31
|
+
Generates and returns a dictionary of affinity and anti-affinity groups
|
|
32
|
+
based on the provided data.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self, proxlb_data: Dict[str, Any]):
|
|
36
|
+
"""
|
|
37
|
+
Initializes the Groups class with the provided ProxLB data.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
proxlb_data (Dict[str, Any]): The data required for balancing VMs and CTs.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
@staticmethod
|
|
44
|
+
def get_groups(guests: Dict[str, Any], nodes: Dict[str, Any]) -> Dict[str, Any]:
|
|
45
|
+
"""
|
|
46
|
+
Generates and returns a dictionary of affinity and anti-affinity groups based on the provided data.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
guests (Dict[str, Any]): A dictionary containing the guest data.
|
|
50
|
+
nodes (Dict[str, Any]): A dictionary containing the nodes data.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
Dict[str, Any]: A dictionary containing the created groups that includes:
|
|
54
|
+
* Affinity groups (or a randon and uniq group)
|
|
55
|
+
* Anti-affinity groups
|
|
56
|
+
* A list of guests that are currently placed on a node which
|
|
57
|
+
is defined to be in maintenance.
|
|
58
|
+
"""
|
|
59
|
+
logger.debug("Starting: get_groups.")
|
|
60
|
+
groups = {'groups': {'affinity': {}, 'anti_affinity': {}, 'maintenance': []}}
|
|
61
|
+
|
|
62
|
+
for guest_name, guest_meta in guests["guests"].items():
|
|
63
|
+
# Create affinity grouping
|
|
64
|
+
# Use an affinity group if available for the guest
|
|
65
|
+
if len(guest_meta["affinity_groups"]) > 0:
|
|
66
|
+
for affinity_group in guest_meta["affinity_groups"]:
|
|
67
|
+
group_name = affinity_group
|
|
68
|
+
logger.debug(f'Affinity group {affinity_group} for {guest_name} will be used.')
|
|
69
|
+
else:
|
|
70
|
+
# Generate a random uniq group name for the guest if
|
|
71
|
+
# the guest does not belong to any affinity group
|
|
72
|
+
random_group = Helper.get_uuid_string()
|
|
73
|
+
group_name = random_group
|
|
74
|
+
logger.debug(f'Random uniq group {random_group} for {guest_name} will be used.')
|
|
75
|
+
|
|
76
|
+
if not groups["groups"]["affinity"].get(group_name, False):
|
|
77
|
+
# Create group template with initial guest meta information
|
|
78
|
+
groups["groups"]["affinity"][group_name] = {}
|
|
79
|
+
groups["groups"]["affinity"][group_name]["guests"] = []
|
|
80
|
+
groups["groups"]["affinity"][group_name]["guests"].append(guest_name)
|
|
81
|
+
groups["groups"]["affinity"][group_name]["counter"] = 1
|
|
82
|
+
# Create groups resource template by the guests resources
|
|
83
|
+
groups["groups"]["affinity"][group_name]["cpu_total"] = guest_meta["cpu_total"]
|
|
84
|
+
groups["groups"]["affinity"][group_name]["cpu_used"] = guest_meta["cpu_used"]
|
|
85
|
+
groups["groups"]["affinity"][group_name]["memory_total"] = guest_meta["memory_total"]
|
|
86
|
+
groups["groups"]["affinity"][group_name]["memory_used"] = guest_meta["cpu_used"]
|
|
87
|
+
groups["groups"]["affinity"][group_name]["disk_total"] = guest_meta["disk_total"]
|
|
88
|
+
groups["groups"]["affinity"][group_name]["disk_used"] = guest_meta["cpu_used"]
|
|
89
|
+
else:
|
|
90
|
+
# Update group templates by guest meta information
|
|
91
|
+
groups["groups"]["affinity"][group_name]["guests"].append(guest_name)
|
|
92
|
+
groups["groups"]["affinity"][group_name]["counter"] += 1
|
|
93
|
+
# Update group resources by guest resources
|
|
94
|
+
groups["groups"]["affinity"][group_name]["cpu_total"] += guest_meta["cpu_total"]
|
|
95
|
+
groups["groups"]["affinity"][group_name]["cpu_used"] += guest_meta["cpu_used"]
|
|
96
|
+
groups["groups"]["affinity"][group_name]["memory_total"] += guest_meta["memory_total"]
|
|
97
|
+
groups["groups"]["affinity"][group_name]["memory_used"] += guest_meta["cpu_used"]
|
|
98
|
+
groups["groups"]["affinity"][group_name]["disk_total"] += guest_meta["disk_total"]
|
|
99
|
+
groups["groups"]["affinity"][group_name]["disk_used"] += guest_meta["cpu_used"]
|
|
100
|
+
|
|
101
|
+
# Create anti-affinity grouping
|
|
102
|
+
if len(guest_meta["anti_affinity_groups"]) > 0:
|
|
103
|
+
for anti_affinity_group in guest_meta["anti_affinity_groups"]:
|
|
104
|
+
anti_affinity_group_name = anti_affinity_group
|
|
105
|
+
logger.debug(f'Anti-affinity group {anti_affinity_group_name} for {guest_name} will be used.')
|
|
106
|
+
|
|
107
|
+
if not groups["groups"]["anti_affinity"].get(anti_affinity_group_name, False):
|
|
108
|
+
groups["groups"]["anti_affinity"][anti_affinity_group_name] = {}
|
|
109
|
+
groups["groups"]["anti_affinity"][anti_affinity_group_name]["guests"] = []
|
|
110
|
+
groups["groups"]["anti_affinity"][anti_affinity_group_name]["guests"].append(guest_name)
|
|
111
|
+
groups["groups"]["anti_affinity"][anti_affinity_group_name]["counter"] = 1
|
|
112
|
+
groups["groups"]["anti_affinity"][anti_affinity_group_name]["used_nodes"] = []
|
|
113
|
+
else:
|
|
114
|
+
groups["groups"]["anti_affinity"][anti_affinity_group_name]["guests"].append(guest_name)
|
|
115
|
+
groups["groups"]["anti_affinity"][anti_affinity_group_name]["counter"] += 1
|
|
116
|
+
|
|
117
|
+
# Create grouping of guests that are currently located on nodes that are
|
|
118
|
+
# marked as in maintenance and must be migrated
|
|
119
|
+
if nodes["nodes"][guest_meta["node_current"]]["maintenance"]:
|
|
120
|
+
logger.debug(f'{guest_name} will be migrated to another node because the underlying node {guest_meta["node_current"]} is defined to be in maintenance.')
|
|
121
|
+
groups["groups"]["maintenance"].append(guest_name)
|
|
122
|
+
|
|
123
|
+
logger.debug("Finished: get_groups.")
|
|
124
|
+
return groups
|
proxlb/models/guests.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The Guests class retrieves all running guests on the Proxmox cluster across all available nodes.
|
|
3
|
+
It handles both VM and CT guest types, collecting their resource metrics.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
|
7
|
+
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
|
8
|
+
__license__ = "GPL-3.0"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
from typing import Dict, Any
|
|
12
|
+
from utils.logger import SystemdLogger
|
|
13
|
+
from models.pools import Pools
|
|
14
|
+
from models.ha_rules import HaRules
|
|
15
|
+
from models.tags import Tags
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
logger = SystemdLogger()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Guests:
|
|
22
|
+
"""
|
|
23
|
+
The Guests class retrieves all running guests on the Proxmox cluster across all available nodes.
|
|
24
|
+
It handles both VM and CT guest types, collecting their resource metrics.
|
|
25
|
+
|
|
26
|
+
Methods:
|
|
27
|
+
__init__:
|
|
28
|
+
Initializes the Guests class.
|
|
29
|
+
|
|
30
|
+
get_guests(proxmox_api: any, nodes: Dict[str, Any]) -> Dict[str, Any]:
|
|
31
|
+
Retrieves metrics for all running guests (both VMs and CTs) across all nodes in the Proxmox cluster.
|
|
32
|
+
It collects resource metrics such as CPU, memory, and disk usage, as well as tags and affinity/anti-affinity groups.
|
|
33
|
+
"""
|
|
34
|
+
def __init__(self):
|
|
35
|
+
"""
|
|
36
|
+
Initializes the Guests class with the provided ProxLB data.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
@staticmethod
|
|
40
|
+
def get_guests(proxmox_api: any, pools: Dict[str, Any], ha_rules: Dict[str, Any], nodes: Dict[str, Any], meta: Dict[str, Any], proxlb_config: Dict[str, Any]) -> Dict[str, Any]:
|
|
41
|
+
"""
|
|
42
|
+
Get metrics of all guests in a Proxmox cluster.
|
|
43
|
+
|
|
44
|
+
This method retrieves metrics for all running guests (both VMs and CTs) across all nodes in the Proxmox cluster.
|
|
45
|
+
It iterates over each node and collects resource metrics for each running guest, including CPU, memory, and disk usage.
|
|
46
|
+
Additionally, it retrieves tags and affinity/anti-affinity groups for each guest.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
proxmox_api (any): The Proxmox API client instance.
|
|
50
|
+
pools (Dict[str, Any]): A dictionary containing information about the pools in the Proxmox cluster.
|
|
51
|
+
ha_rules (Dict[str, Any]): A dictionary containing information about the HA rules in the
|
|
52
|
+
nodes (Dict[str, Any]): A dictionary containing information about the nodes in the Proxmox cluster.
|
|
53
|
+
meta (Dict[str, Any]): A dictionary containing metadata information.
|
|
54
|
+
proxmox_config (Dict[str, Any]): A dictionary containing the ProxLB configuration.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
Dict[str, Any]: A dictionary containing metrics and information for all running guests.
|
|
58
|
+
"""
|
|
59
|
+
logger.debug("Starting: get_guests.")
|
|
60
|
+
guests = {"guests": {}}
|
|
61
|
+
|
|
62
|
+
# Guest objects are always only in the scope of a node.
|
|
63
|
+
# Therefore, we need to iterate over all nodes to get all guests.
|
|
64
|
+
for node in nodes['nodes'].keys():
|
|
65
|
+
|
|
66
|
+
# VM objects: Iterate over all VMs on the current node by the qemu API object.
|
|
67
|
+
# Unlike the nodes we need to keep them even when being ignored to create proper
|
|
68
|
+
# resource metrics for rebalancing to ensure that we do not overprovisiong the node.
|
|
69
|
+
for guest in proxmox_api.nodes(node).qemu.get():
|
|
70
|
+
if guest['status'] == 'running':
|
|
71
|
+
guests['guests'][guest['name']] = {}
|
|
72
|
+
guests['guests'][guest['name']]['name'] = guest['name']
|
|
73
|
+
guests['guests'][guest['name']]['cpu_total'] = int(guest['cpus'])
|
|
74
|
+
guests['guests'][guest['name']]['cpu_used'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', None)
|
|
75
|
+
guests['guests'][guest['name']]['cpu_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'some')
|
|
76
|
+
guests['guests'][guest['name']]['cpu_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'full')
|
|
77
|
+
guests['guests'][guest['name']]['cpu_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'some', spikes=True)
|
|
78
|
+
guests['guests'][guest['name']]['cpu_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'full', spikes=True)
|
|
79
|
+
guests['guests'][guest['name']]['cpu_pressure_hot'] = False
|
|
80
|
+
guests['guests'][guest['name']]['memory_total'] = guest['maxmem']
|
|
81
|
+
guests['guests'][guest['name']]['memory_used'] = guest['mem']
|
|
82
|
+
guests['guests'][guest['name']]['memory_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'some')
|
|
83
|
+
guests['guests'][guest['name']]['memory_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'full')
|
|
84
|
+
guests['guests'][guest['name']]['memory_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'some', spikes=True)
|
|
85
|
+
guests['guests'][guest['name']]['memory_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'full', spikes=True)
|
|
86
|
+
guests['guests'][guest['name']]['memory_pressure_hot'] = False
|
|
87
|
+
guests['guests'][guest['name']]['disk_total'] = guest['maxdisk']
|
|
88
|
+
guests['guests'][guest['name']]['disk_used'] = guest['disk']
|
|
89
|
+
guests['guests'][guest['name']]['disk_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'some')
|
|
90
|
+
guests['guests'][guest['name']]['disk_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'full')
|
|
91
|
+
guests['guests'][guest['name']]['disk_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'some', spikes=True)
|
|
92
|
+
guests['guests'][guest['name']]['disk_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'full', spikes=True)
|
|
93
|
+
guests['guests'][guest['name']]['disk_pressure_hot'] = False
|
|
94
|
+
guests['guests'][guest['name']]['id'] = guest['vmid']
|
|
95
|
+
guests['guests'][guest['name']]['node_current'] = node
|
|
96
|
+
guests['guests'][guest['name']]['node_target'] = node
|
|
97
|
+
guests['guests'][guest['name']]['processed'] = False
|
|
98
|
+
guests['guests'][guest['name']]['pressure_hot'] = False
|
|
99
|
+
guests['guests'][guest['name']]['tags'] = Tags.get_tags_from_guests(proxmox_api, node, guest['vmid'], 'vm')
|
|
100
|
+
guests['guests'][guest['name']]['pools'] = Pools.get_pools_for_guest(guest['name'], pools)
|
|
101
|
+
guests['guests'][guest['name']]['ha_rules'] = HaRules.get_ha_rules_for_guest(guest['name'], ha_rules, guest['vmid'])
|
|
102
|
+
guests['guests'][guest['name']]['affinity_groups'] = Tags.get_affinity_groups(guests['guests'][guest['name']]['tags'], guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
|
103
|
+
guests['guests'][guest['name']]['anti_affinity_groups'] = Tags.get_anti_affinity_groups(guests['guests'][guest['name']]['tags'], guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
|
104
|
+
guests['guests'][guest['name']]['ignore'] = Tags.get_ignore(guests['guests'][guest['name']]['tags'])
|
|
105
|
+
guests['guests'][guest['name']]['node_relationships'] = Tags.get_node_relationships(guests['guests'][guest['name']]['tags'], nodes, guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
|
106
|
+
guests['guests'][guest['name']]['node_relationships_strict'] = Pools.get_pool_node_affinity_strictness(proxlb_config, guests['guests'][guest['name']]['pools'])
|
|
107
|
+
guests['guests'][guest['name']]['type'] = 'vm'
|
|
108
|
+
|
|
109
|
+
logger.debug(f"Resources of Guest {guest['name']} (type VM) added: {guests['guests'][guest['name']]}")
|
|
110
|
+
else:
|
|
111
|
+
logger.debug(f'Metric for VM {guest["name"]} ignored because VM is not running.')
|
|
112
|
+
|
|
113
|
+
# CT objects: Iterate over all VMs on the current node by the lxc API object.
|
|
114
|
+
# Unlike the nodes we need to keep them even when being ignored to create proper
|
|
115
|
+
# resource metrics for rebalancing to ensure that we do not overprovisiong the node.
|
|
116
|
+
for guest in proxmox_api.nodes(node).lxc.get():
|
|
117
|
+
if guest['status'] == 'running':
|
|
118
|
+
guests['guests'][guest['name']] = {}
|
|
119
|
+
guests['guests'][guest['name']]['name'] = guest['name']
|
|
120
|
+
guests['guests'][guest['name']]['cpu_total'] = int(guest['cpus'])
|
|
121
|
+
guests['guests'][guest['name']]['cpu_used'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', None)
|
|
122
|
+
guests['guests'][guest['name']]['cpu_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'some')
|
|
123
|
+
guests['guests'][guest['name']]['cpu_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'full')
|
|
124
|
+
guests['guests'][guest['name']]['cpu_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'some', spikes=True)
|
|
125
|
+
guests['guests'][guest['name']]['cpu_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'cpu', 'full', spikes=True)
|
|
126
|
+
guests['guests'][guest['name']]['cpu_pressure_hot'] = False
|
|
127
|
+
guests['guests'][guest['name']]['memory_total'] = guest['maxmem']
|
|
128
|
+
guests['guests'][guest['name']]['memory_used'] = guest['mem']
|
|
129
|
+
guests['guests'][guest['name']]['memory_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'some')
|
|
130
|
+
guests['guests'][guest['name']]['memory_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'full')
|
|
131
|
+
guests['guests'][guest['name']]['memory_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'some', spikes=True)
|
|
132
|
+
guests['guests'][guest['name']]['memory_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'memory', 'full', spikes=True)
|
|
133
|
+
guests['guests'][guest['name']]['memory_pressure_hot'] = False
|
|
134
|
+
guests['guests'][guest['name']]['disk_total'] = guest['maxdisk']
|
|
135
|
+
guests['guests'][guest['name']]['disk_used'] = guest['disk']
|
|
136
|
+
guests['guests'][guest['name']]['disk_pressure_some_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'some')
|
|
137
|
+
guests['guests'][guest['name']]['disk_pressure_full_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'full')
|
|
138
|
+
guests['guests'][guest['name']]['disk_pressure_some_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'some', spikes=True)
|
|
139
|
+
guests['guests'][guest['name']]['disk_pressure_full_spikes_percent'] = Guests.get_guest_rrd_data(proxmox_api, node, guest['vmid'], guest['name'], 'disk', 'full', spikes=True)
|
|
140
|
+
guests['guests'][guest['name']]['disk_pressure_hot'] = False
|
|
141
|
+
guests['guests'][guest['name']]['id'] = guest['vmid']
|
|
142
|
+
guests['guests'][guest['name']]['node_current'] = node
|
|
143
|
+
guests['guests'][guest['name']]['node_target'] = node
|
|
144
|
+
guests['guests'][guest['name']]['processed'] = False
|
|
145
|
+
guests['guests'][guest['name']]['pressure_hot'] = False
|
|
146
|
+
guests['guests'][guest['name']]['tags'] = Tags.get_tags_from_guests(proxmox_api, node, guest['vmid'], 'ct')
|
|
147
|
+
guests['guests'][guest['name']]['pools'] = Pools.get_pools_for_guest(guest['name'], pools)
|
|
148
|
+
guests['guests'][guest['name']]['ha_rules'] = HaRules.get_ha_rules_for_guest(guest['name'], ha_rules, guest['vmid'])
|
|
149
|
+
guests['guests'][guest['name']]['affinity_groups'] = Tags.get_affinity_groups(guests['guests'][guest['name']]['tags'], guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
|
150
|
+
guests['guests'][guest['name']]['anti_affinity_groups'] = Tags.get_anti_affinity_groups(guests['guests'][guest['name']]['tags'], guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
|
151
|
+
guests['guests'][guest['name']]['ignore'] = Tags.get_ignore(guests['guests'][guest['name']]['tags'])
|
|
152
|
+
guests['guests'][guest['name']]['node_relationships'] = Tags.get_node_relationships(guests['guests'][guest['name']]['tags'], nodes, guests['guests'][guest['name']]['pools'], guests['guests'][guest['name']]['ha_rules'], proxlb_config)
|
|
153
|
+
guests['guests'][guest['name']]['node_relationships_strict'] = Pools.get_pool_node_affinity_strictness(proxlb_config, guests['guests'][guest['name']]['pools'])
|
|
154
|
+
guests['guests'][guest['name']]['type'] = 'ct'
|
|
155
|
+
|
|
156
|
+
logger.debug(f"Resources of Guest {guest['name']} (type CT) added: {guests['guests'][guest['name']]}")
|
|
157
|
+
else:
|
|
158
|
+
logger.debug(f'Metric for CT {guest["name"]} ignored because CT is not running.')
|
|
159
|
+
|
|
160
|
+
logger.debug("Finished: get_guests.")
|
|
161
|
+
return guests
|
|
162
|
+
|
|
163
|
+
@staticmethod
|
|
164
|
+
def get_guest_rrd_data(proxmox_api, node_name: str, vm_id: int, vm_name: str, object_name: str, object_type: str, spikes=False) -> float:
|
|
165
|
+
"""
|
|
166
|
+
Retrieves the rrd data metrics for a specific resource (CPU, memory, disk) of a guest VM or CT.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
proxmox_api (Any): The Proxmox API client instance.
|
|
170
|
+
node_name (str): The name of the node hosting the guest.
|
|
171
|
+
vm_id (int): The ID of the guest VM or CT.
|
|
172
|
+
vm_name (str): The name of the guest VM or CT.
|
|
173
|
+
object_name (str): The resource type to query (e.g., 'cpu', 'memory', 'disk').
|
|
174
|
+
object_type (str, optional): The pressure type ('some', 'full') or None for average usage.
|
|
175
|
+
spikes (bool, optional): Whether to consider spikes in the calculation. Defaults to False.
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
float: The calculated average usage value for the specified resource.
|
|
179
|
+
"""
|
|
180
|
+
logger.debug("Starting: get_guest_rrd_data.")
|
|
181
|
+
time.sleep(0.1)
|
|
182
|
+
|
|
183
|
+
try:
|
|
184
|
+
if spikes:
|
|
185
|
+
logger.debug(f"Getting spike RRD data for {object_name} from guest: {vm_name}.")
|
|
186
|
+
guest_data_rrd = proxmox_api.nodes(node_name).qemu(vm_id).rrddata.get(timeframe="hour", cf="MAX")
|
|
187
|
+
else:
|
|
188
|
+
logger.debug(f"Getting average RRD data for {object_name} from guest: {vm_name}.")
|
|
189
|
+
guest_data_rrd = proxmox_api.nodes(node_name).qemu(vm_id).rrddata.get(timeframe="hour", cf="AVERAGE")
|
|
190
|
+
except Exception:
|
|
191
|
+
logger.error(f"Failed to retrieve RRD data for guest: {vm_name} (ID: {vm_id}) on node: {node_name}. Using 0.0 as value.")
|
|
192
|
+
logger.debug("Finished: get_guest_rrd_data.")
|
|
193
|
+
return float(0.0)
|
|
194
|
+
|
|
195
|
+
if object_type:
|
|
196
|
+
|
|
197
|
+
lookup_key = f"pressure{object_name}{object_type}"
|
|
198
|
+
if spikes:
|
|
199
|
+
# RRD data is collected every minute, so we look at the last 6 entries
|
|
200
|
+
# and take the maximum value to represent the spike
|
|
201
|
+
logger.debug(f"Getting RRD data (spike: {spikes}) of pressure for {object_name} {object_type} from guest: {vm_name}.")
|
|
202
|
+
rrd_data_value = [row.get(lookup_key) for row in guest_data_rrd if row.get(lookup_key) is not None]
|
|
203
|
+
rrd_data_value = max(rrd_data_value[-6:], default=0.0)
|
|
204
|
+
else:
|
|
205
|
+
# Calculate the average value from the RRD data entries
|
|
206
|
+
logger.debug(f"Getting RRD data (spike: {spikes}) of pressure for {object_name} {object_type} from guest: {vm_name}.")
|
|
207
|
+
rrd_data_value = sum(entry.get(lookup_key, 0.0) for entry in guest_data_rrd) / len(guest_data_rrd)
|
|
208
|
+
|
|
209
|
+
else:
|
|
210
|
+
logger.debug(f"Getting RRD data of cpu usage from guest: {vm_name}.")
|
|
211
|
+
rrd_data_value = sum(entry.get("cpu", 0.0) for entry in guest_data_rrd) / len(guest_data_rrd)
|
|
212
|
+
|
|
213
|
+
logger.debug(f"RRD data (spike: {spikes}) for {object_name} from guest: {vm_name}: {rrd_data_value}")
|
|
214
|
+
logger.debug("Finished: get_guest_rrd_data.")
|
|
215
|
+
return rrd_data_value
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The HaRules class retrieves all HA rules defined on a Proxmox cluster
|
|
3
|
+
including their affinity settings and member resources.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
__author__ = "Florian Paul Azim Hoberg <gyptazy>"
|
|
7
|
+
__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
|
|
8
|
+
__license__ = "GPL-3.0"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
from typing import Dict, Any
|
|
12
|
+
from utils.logger import SystemdLogger
|
|
13
|
+
|
|
14
|
+
logger = SystemdLogger()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class HaRules:
|
|
18
|
+
"""
|
|
19
|
+
The HaRules class retrieves all HA rules defined on a Proxmox cluster
|
|
20
|
+
including their (anti)a-ffinity settings and member resources and translates
|
|
21
|
+
them into a ProxLB usable format.
|
|
22
|
+
|
|
23
|
+
Methods:
|
|
24
|
+
__init__:
|
|
25
|
+
Initializes the HaRules class.
|
|
26
|
+
|
|
27
|
+
get_ha_rules(proxmox_api: any) -> Dict[str, Any]:
|
|
28
|
+
Retrieve HA rule definitions from the Proxmox cluster.
|
|
29
|
+
Returns a dict with a top-level "ha_rules" mapping each rule id to
|
|
30
|
+
{"rule": <rule_id>, "type": <affinity_type>, "members": [<resource_ids>...]}.
|
|
31
|
+
Converts affinity settings to descriptive format (affinity or anti-affinity).
|
|
32
|
+
"""
|
|
33
|
+
def __init__(self):
|
|
34
|
+
"""
|
|
35
|
+
Initializes the HA Rules class with the provided ProxLB data.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
@staticmethod
|
|
39
|
+
def get_ha_rules(proxmox_api: any, meta: dict) -> Dict[str, Any]:
|
|
40
|
+
"""
|
|
41
|
+
Retrieve all HA rules from a Proxmox cluster.
|
|
42
|
+
|
|
43
|
+
Queries the Proxmox API for HA rule definitions and returns a dictionary
|
|
44
|
+
containing each rule's id, affinity type, and member resources (VM/CT IDs).
|
|
45
|
+
This function processes rule affinity settings and converts them to a more
|
|
46
|
+
descriptive format (affinity or anti-affinity).
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
proxmox_api (any): Proxmox API client instance.
|
|
50
|
+
meta (dict): The metadata dictionary containing cluster information.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
Dict[str, Any]: Dictionary with a top-level "ha_rules" key mapping rule id
|
|
54
|
+
to {"rule": <rule_id>, "type": <affinity_type>, "members": [<resource_ids>...]}.
|
|
55
|
+
"""
|
|
56
|
+
logger.debug("Starting: get_ha_rules.")
|
|
57
|
+
ha_rules = {"ha_rules": {}}
|
|
58
|
+
|
|
59
|
+
# If any node is non PVE 9, skip fetching HA rules as they are unsupported
|
|
60
|
+
if meta["meta"]["cluster_non_pve9"]:
|
|
61
|
+
logger.debug("Skipping HA rule retrieval as non Proxmox VE 9 systems detected.")
|
|
62
|
+
return ha_rules
|
|
63
|
+
else:
|
|
64
|
+
logger.debug("Cluster running Proxmox VE 9 or newer, proceeding with HA rule retrieval.")
|
|
65
|
+
|
|
66
|
+
for rule in proxmox_api.cluster.ha.rules.get():
|
|
67
|
+
|
|
68
|
+
# Skip disabled rules (disable key exists AND is truthy)
|
|
69
|
+
if rule.get("disable", 0):
|
|
70
|
+
logger.debug(f"Skipping ha-rule: {rule['rule']} of type {rule['type']} affecting guests: {rule['resources']}. Rule is disabled.")
|
|
71
|
+
continue
|
|
72
|
+
|
|
73
|
+
# Create a resource list by splitting on commas and stripping whitespace containing
|
|
74
|
+
# the VM and CT IDs that are part of this HA rule
|
|
75
|
+
resources_list_guests = [int(r.split(":")[1]) for r in rule["resources"].split(",") if r.strip()]
|
|
76
|
+
|
|
77
|
+
# Convert the affinity field to a more descriptive type
|
|
78
|
+
if rule.get("affinity", None) == "negative":
|
|
79
|
+
affinity_type = "anti-affinity"
|
|
80
|
+
else:
|
|
81
|
+
affinity_type = "affinity"
|
|
82
|
+
|
|
83
|
+
# Create affected nodes list
|
|
84
|
+
resources_list_nodes = []
|
|
85
|
+
if rule.get("nodes", None):
|
|
86
|
+
resources_list_nodes = [n for n in rule["nodes"].split(",") if n]
|
|
87
|
+
|
|
88
|
+
# Create the ha_rule element
|
|
89
|
+
ha_rules['ha_rules'][rule['rule']] = {}
|
|
90
|
+
ha_rules['ha_rules'][rule['rule']]['rule'] = rule['rule']
|
|
91
|
+
ha_rules['ha_rules'][rule['rule']]['type'] = affinity_type
|
|
92
|
+
ha_rules['ha_rules'][rule['rule']]['nodes'] = resources_list_nodes
|
|
93
|
+
ha_rules['ha_rules'][rule['rule']]['members'] = resources_list_guests
|
|
94
|
+
|
|
95
|
+
logger.debug(f"Got ha-rule: {rule['rule']} as type {affinity_type} affecting guests: {rule['resources']}")
|
|
96
|
+
|
|
97
|
+
logger.debug("Finished: ha_rules.")
|
|
98
|
+
return ha_rules
|
|
99
|
+
|
|
100
|
+
@staticmethod
|
|
101
|
+
def get_ha_rules_for_guest(guest_name: str, ha_rules: Dict[str, Any], vm_id: int) -> Dict[str, Any]:
|
|
102
|
+
"""
|
|
103
|
+
Return the list of HA rules that include the given guest.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
guest_name (str): Name of the VM or CT to look up.
|
|
107
|
+
ha_rules (Dict[str, Any]): HA rules structure as returned by get_ha_rules(),
|
|
108
|
+
expected to contain a top-level "ha_rules" mapping each rule id to
|
|
109
|
+
{"rule": <rule_id>, "type": <affinity_type>, "members": [<resource_ids>...]}.
|
|
110
|
+
vm_id (int): VM or CT ID of the guest.
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
list: IDs of HA rules the guest is a member of (empty list if none).
|
|
114
|
+
"""
|
|
115
|
+
logger.debug("Starting: get_ha_rules_for_guest.")
|
|
116
|
+
guest_ha_rules = []
|
|
117
|
+
|
|
118
|
+
for rule in ha_rules["ha_rules"].values():
|
|
119
|
+
if vm_id in rule.get("members", []):
|
|
120
|
+
logger.debug(f"Guest: {guest_name} (VMID: {vm_id}) is member of HA Rule: {rule['rule']}.")
|
|
121
|
+
guest_ha_rules.append(rule)
|
|
122
|
+
else:
|
|
123
|
+
logger.debug(f"Guest: {guest_name} (VMID: {vm_id}) is NOT member of HA Rule: {rule['rule']}.")
|
|
124
|
+
|
|
125
|
+
logger.debug("Finished: get_ha_rules_for_guest.")
|
|
126
|
+
return guest_ha_rules
|