proxlb 1.1.12.dev4__py3-none-any.whl

etc/proxlb/proxlb_example.yaml

@@ -0,0 +1,90 @@
+proxmox_api:
+  hosts: ['virt01.example.com', '10.10.10.10', 'fe01:bad:code::cafe']
+  user: root@pam
+  pass: crazyPassw0rd!
+  # API Token method
+  # token_id: proxlb
+  # token_secret: 430e308f-1337-1337-beef-1337beefcafe
+  ssl_verification: True
+  timeout: 10
+  # API Connection retries
+  # retries: 1
+  # wait_time: 1
+
+proxmox_cluster:
+  maintenance_nodes: ['virt66.example.com']
+  ignore_nodes: []
+  overprovisioning: True
+
+balancing:
+  enable: True
+  enforce_affinity: False
+  enforce_pinning: False
+  parallel: False
+  # If running parallel job, you can define
+  # the amount of prallel jobs (default: 5)
+  parallel_jobs: 1
+  live: True
+  with_local_disks: True
+  with_conntrack_state: True
+  balance_types: ['vm', 'ct']         # 'vm' | 'ct'
+  max_job_validation: 1800            # Maximum time (in seconds) a job validation may take
+  memory_threshold: 75                # Optional: Maximum threshold (in percent) to trigger balancing actions
+  cpu_threshold: 75                   # Optional: Maximum threshold (in percent) to trigger balancing actions
+  balanciness: 5                      # Maximum delta of resource usage between highest and lowest usage node
+  method: memory                      # 'memory' | 'cpu' | 'disk'
+  mode: used                          # 'assigned' | 'used' | 'psi'
+  balance_larger_guests_first: False  # Option to prioritize balancing of larger or smaller guests first
+  node_resource_reserve:              # Optional: Define resource reservations for nodes (in GB)
+    defaults:                         # Default reservation values applying to all nodes (unless explicitly overridden)
+      memory: 4                       # Default: 4 GB memory reserved per node
+    node01:                           # Specific node reservation override for node 'node01'
+      memory: 6                       # Specific: 6 GB memory reserved for node 'node01'
+  # # PSI thresholds only apply when using mode 'psi'
+  # psi:
+  #   nodes:
+  #     memory:
+  #       pressure_full: 0.20
+  #       pressure_some: 0.20
+  #       pressure_spikes: 1.00
+  #     cpu:
+  #       pressure_full: 0.20
+  #       pressure_some: 0.20
+  #       pressure_spikes: 1.00
+  #     disk:
+  #       pressure_full: 0.20
+  #       pressure_some: 0.20
+  #       pressure_spikes: 1.00
+  #   guests:
+  #     memory:
+  #       pressure_full: 0.20
+  #       pressure_some: 0.20
+  #       pressure_spikes: 1.00
+  #     cpu:
+  #       pressure_full: 0.20
+  #       pressure_some: 0.20
+  #       pressure_spikes: 1.00
+  #     disk:
+  #       pressure_full: 0.20
+  #       pressure_some: 0.20
+  #       pressure_spikes: 1.00
+  pools:                              # Optional: Define affinity/anti-affinity rules per pool
+    dev:                              # Pool name: dev
+      type: affinity                  # Type: affinity (keeping VMs together)
+    de-nbg01-db:                      # Pool name: de-nbg01-db
+      type: anti-affinity             # Type: anti-affinity (spreading VMs apart)
+      pin:                            # Define a pinning og guests to specific node(s)
+        - virt66
+        - virt77
+      strict: False                   # Disable strict mode of node pinning for this pool
+
+service:
+  daemon: True
+  schedule:
+    interval: 12
+    format: hours
+  delay:
+    enable: False
+    time: 1
+    format: hours
+  log_level: INFO

etc/systemd/system/proxlb.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=ProxLB - A loadbalancer for Proxmox clusters
+After=network-online.target pveproxy.service
+Wants=network-online.target pveproxy.service
+
+[Service]
+ExecStart=python3 /usr/lib/python3/dist-packages/proxlb/main.py -c /etc/proxlb/proxlb.yaml
+User=plb
+ExecReload=/bin/kill -HUP $MAINPID
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

proxlb/main.py

@@ -0,0 +1,120 @@
+"""
+ProxLB is a load balancing tool for Proxmox Virtual Environment (PVE) clusters.
+It connects to the Proxmox API, retrieves information about nodes, guests, and groups,
+and performs calculations to determine the optimal distribution of resources across the
+cluster. The tool supports daemon mode for continuous operation and can log metrics and
+perform balancing actions based on the configuration provided. It also includes a CLI
+parser for handling command-line arguments and a custom logger for systemd integration.
+"""
+
+__author__ = "Florian Paul Azim Hoberg <gyptazy>"
+__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
+__license__ = "GPL-3.0"
+
+
+import logging
+import signal
+from utils.logger import SystemdLogger
+from utils.cli_parser import CliParser
+from utils.config_parser import ConfigParser
+from utils.proxmox_api import ProxmoxApi
+from models.nodes import Nodes
+from models.features import Features
+from models.guests import Guests
+from models.groups import Groups
+from models.calculations import Calculations
+from models.balancing import Balancing
+from models.pools import Pools
+from models.ha_rules import HaRules
+from utils.helper import Helper
+
+
+def main():
+    """
+    ProxLB main function
+    """
+    # Initialize logging handler
+    logger = SystemdLogger(level=logging.INFO)
+
+    # Initialize handlers
+    signal.signal(signal.SIGHUP, Helper.handler_sighup)
+    signal.signal(signal.SIGINT, Helper.handler_sigint)
+
+    # Parses arguments passed from the CLI
+    cli_parser = CliParser()
+    cli_args = cli_parser.parse_args()
+    Helper.get_version(cli_args.version)
+
+    # Parse ProxLB config file
+    config_parser = ConfigParser(cli_args.config)
+    proxlb_config = config_parser.get_config()
+
+    # Update log level from config and fallback to INFO if not defined
+    logger.set_log_level(proxlb_config.get('service', {}).get('log_level', 'INFO'))
+
+    # Validate of an optional service delay
+    Helper.get_service_delay(proxlb_config)
+
+    # Connect to Proxmox API & create API object
+    proxmox_api = ProxmoxApi(proxlb_config)
+
+    # Overwrite password after creating the API object
+    proxlb_config["proxmox_api"]["pass"] = "********"
+
+    while True:
+
+        # Validate if reload signal was sent during runtime
+        # and reload the ProxLB configuration and adjust log level
+        if Helper.proxlb_reload:
+            logger.info("Reloading ProxLB configuration.")
+            proxlb_config = config_parser.get_config()
+            logger.set_log_level(proxlb_config.get('service', {}).get('log_level', 'INFO'))
+            Helper.proxlb_reload = False
+
+        # Get all required objects from the Proxmox cluster
+        meta = {"meta": proxlb_config}
+        nodes = Nodes.get_nodes(proxmox_api, proxlb_config)
+        meta = Features.validate_any_non_pve9_node(meta, nodes)
+        pools = Pools.get_pools(proxmox_api)
+        ha_rules = HaRules.get_ha_rules(proxmox_api, meta)
+        guests = Guests.get_guests(proxmox_api, pools, ha_rules, nodes, meta, proxlb_config)
+        groups = Groups.get_groups(guests, nodes)
+
+        # Merge obtained objects from the Proxmox cluster for further usage
+        proxlb_data = {**meta, **nodes, **guests, **pools, **ha_rules, **groups}
+        Helper.log_node_metrics(proxlb_data)
+
+        # Validate usable features by PVE versions
+        Features.validate_available_features(proxlb_data)
+
+        # Update the initial node resource assignments
+        # by the previously created groups.
+        Calculations.set_node_assignments(proxlb_data)
+        Helper.log_node_metrics(proxlb_data, init=False)
+        Calculations.set_node_hot(proxlb_data)
+        Calculations.set_guest_hot(proxlb_data)
+        target = Calculations.get_most_free_node(proxlb_data, cli_args.best_node)
+        if target is None:
+            logger.warning("No suitable target node found for balancing. Skipping this run.")
+        else:
+            Calculations.validate_affinity_map(proxlb_data)
+            Calculations.relocate_guests_on_maintenance_nodes(proxlb_data)
+            Calculations.get_balanciness(proxlb_data)
+            Calculations.relocate_guests(proxlb_data)
+            Helper.log_node_metrics(proxlb_data, init=False)
+
+            # Perform balancing actions via Proxmox API
+            if proxlb_data["meta"]["balancing"].get("enable", False):
+                if not cli_args.dry_run:
+                    Balancing(proxmox_api, proxlb_data)
+
+        # Validate if the JSON output should be
+        # printed to stdout
+        Helper.print_json(proxlb_data, cli_args.json)
+        # Validate daemon mode
+        Helper.get_daemon_mode(proxlb_config)
+        logger.debug(f"Finished: __main__")
+
+
+if __name__ == "__main__":
+    main()

proxlb/models/balancing.py

@@ -0,0 +1,270 @@
+"""
+The Balancing class is responsible for processing workloads on Proxmox clusters.
+It processes the previously generated data (held in proxlb_data) and moves guests
+and other supported types across Proxmox clusters based on the defined values by an operator.
+"""
+
+
+__author__ = "Florian Paul Azim Hoberg <gyptazy>"
+__copyright__ = "Copyright (C) 2025 Florian Paul Azim Hoberg (@gyptazy)"
+__license__ = "GPL-3.0"
+
+
+import proxmoxer
+import time
+from itertools import islice
+from utils.logger import SystemdLogger
+from typing import Dict, Any
+
+logger = SystemdLogger()
+
+
+class Balancing:
+    """
+    The balancing class is responsible for processing workloads on Proxmox clusters.
+    The previously generated data (hold in proxlb_data) will processed and guests and
+    other supported types will be moved across Proxmox clusters based on the defined
+    values by an operator.
+
+    Methods:
+    __init__(self, proxmox_api: any, proxlb_data: Dict[str, Any]):
+        Initializes the Balancing class with the provided ProxLB data and initiates the rebalancing
+        process for guests.
+
+    exec_rebalancing_vm(self, proxmox_api: any, proxlb_data: Dict[str, Any], guest_name: str) -> None:
+        Executes the rebalancing of a virtual machine (VM) to a new node within the cluster. Logs the migration
+        process and handles exceptions.
+
+    exec_rebalancing_ct(self, proxmox_api: any, proxlb_data: Dict[str, Any], guest_name: str) -> None:
+        Executes the rebalancing of a container (CT) to a new node within the cluster. Logs the migration
+        process and handles exceptions.
+
+    get_rebalancing_job_status(self, proxmox_api: any, proxlb_data: Dict[str, Any], guest_name: str, guest_current_node: str, job_id: int, retry_counter: int = 1) -> bool:
+        Monitors the status of a rebalancing job on a Proxmox node until it completes or a timeout
+        is reached. Returns True if the job completed successfully, False otherwise.
+    """
+
+    def __init__(self, proxmox_api: any, proxlb_data: Dict[str, Any]):
+        """
+        Initializes the Balancing class with the provided ProxLB data.
+
+        Args:
+            proxmox_api (object): The Proxmox API client instance used to interact with the Proxmox cluster.
+            proxlb_data (dict): A dictionary containing data related to the ProxLB load balancing configuration.
+        """
+        def chunk_dict(data, size):
+            """
+            Splits a dictionary into chunks of a specified size.
+            Args:
+                data (dict): The dictionary to be split into chunks.
+                size (int): The size of each chunk.
+            Yields:
+                dict: A chunk of the original dictionary with the specified size.
+            """
+            logger.debug("Starting: chunk_dict.")
+            it = iter(data.items())
+            for chunk in range(0, len(data), size):
+                yield dict(islice(it, size))
+
+        # Validate if balancing should be performed in parallel or sequentially.
+        # If parallel balancing is enabled, set the number of parallel jobs.
+        parallel_jobs = proxlb_data["meta"]["balancing"].get("parallel_jobs", 5)
+        if not proxlb_data["meta"]["balancing"].get("parallel", False):
+            parallel_jobs = 1
+            logger.debug("Balancing: Parallel balancing is disabled. Running sequentially.")
+        else:
+            logger.debug(f"Balancing: Parallel balancing is enabled. Running with {parallel_jobs} parallel jobs.")
+
+        for chunk in chunk_dict(proxlb_data["guests"], parallel_jobs):
+            jobs_to_wait = []
+
+            for guest_name, guest_meta in chunk.items():
+
+                # Check if the guest's target is not the same as the current node
+                if guest_meta["node_current"] != guest_meta["node_target"]:
+
+                    # Check if the guest is not ignored and perform the balancing
+                    # operation based on the guest type
+                    if not guest_meta["ignore"]:
+                        job_id = None
+
+                        # VM Balancing
+                        if guest_meta["type"] == "vm":
+                            if 'vm' in proxlb_data["meta"]["balancing"].get("balance_types", []):
+                                logger.debug(f"Balancing: Balancing for guest {guest_name} of type VM started.")
+                                job_id = self.exec_rebalancing_vm(proxmox_api, proxlb_data, guest_name)
+                            else:
+                                logger.debug(
+                                    f"Balancing: Balancing for guest {guest_name} will not be performed. "
+                                    "Guest is of type VM which is not included in allowed balancing types.")
+
+                        # CT Balancing
+                        elif guest_meta["type"] == "ct":
+                            if 'ct' in proxlb_data["meta"]["balancing"].get("balance_types", []):
+                                logger.debug(f"Balancing: Balancing for guest {guest_name} of type CT started.")
+                                job_id = self.exec_rebalancing_ct(proxmox_api, proxlb_data, guest_name)
+                            else:
+                                logger.debug(
+                                    f"Balancing: Balancing for guest {guest_name} will not be performed. "
+                                    "Guest is of type CT which is not included in allowed balancing types.")
+
+                        # Just in case we get a new type of guest in the future
+                        else:
+                            logger.critical(f"Balancing: Got unexpected guest type: {guest_meta['type']}. Cannot proceed guest: {guest_meta['name']}.")
+
+                        if job_id:
+                            jobs_to_wait.append((guest_name, guest_meta["node_current"], job_id))
+
+                    else:
+                        logger.debug(f"Balancing: Guest {guest_name} is ignored and will not be rebalanced.")
+                else:
+                    logger.debug(f"Balancing: Guest {guest_name} is already on the target node {guest_meta['node_target']} and will not be rebalanced.")
+
+            # Wait for all jobs in the current chunk to complete
+            for guest_name, node, job_id in jobs_to_wait:
+                if job_id:
+                    self.get_rebalancing_job_status(proxmox_api, proxlb_data, guest_name, node, job_id)
+
+    def exec_rebalancing_vm(self, proxmox_api: any, proxlb_data: Dict[str, Any], guest_name: str) -> None:
+        """
+        Executes the rebalancing of a virtual machine (VM) to a new node within the cluster.
+        This function initiates the migration of a specified VM to a target node as part of the
+        load balancing process. It logs the migration process and handles any exceptions that
+        may occur during the migration.
+        Args:
+            proxmox_api (object): The Proxmox API client instance used to interact with the Proxmox cluster.
+            proxlb_data (dict): A dictionary containing data related to the ProxLB load balancing configuration.
+            guest_name (str): The name of the guest VM to be migrated.
+        Raises:
+            proxmox_api.core.ResourceException: If an error occurs during the migration process.
+        Returns:
+            None
+        """
+        logger.debug("Starting: exec_rebalancing_vm.")
+        guest_id = proxlb_data["guests"][guest_name]["id"]
+        guest_node_current = proxlb_data["guests"][guest_name]["node_current"]
+        guest_node_target = proxlb_data["guests"][guest_name]["node_target"]
+        job_id = None
+
+        if proxlb_data["meta"]["balancing"].get("live", True):
+            online_migration = 1
+        else:
+            online_migration = 0
+
+        if proxlb_data["meta"]["balancing"].get("with_local_disks", True):
+            with_local_disks = 1
+        else:
+            with_local_disks = 0
+
+        migration_options = {
+            'target': guest_node_target,
+            'online': online_migration,
+            'with-local-disks': with_local_disks,
+        }
+
+        # Conntrack state aware migrations are not supported in older
+        # PVE versions, so we should not add it by default.
+        if proxlb_data["meta"]["balancing"].get("with_conntrack_state", True):
+            migration_options['with-conntrack-state'] = 1
+
+        try:
+            logger.info(f"Balancing: Starting to migrate VM guest {guest_name} from {guest_node_current} to {guest_node_target}.")
+            job_id = proxmox_api.nodes(guest_node_current).qemu(guest_id).migrate().post(**migration_options)
+        except proxmoxer.core.ResourceException as proxmox_api_error:
+            logger.critical(f"Balancing: Failed to migrate guest {guest_name} of type VM due to some Proxmox errors. Please check if resource is locked or similar.")
+            logger.debug(f"Balancing: Failed to migrate guest {guest_name} of type VM due to some Proxmox errors: {proxmox_api_error}")
+
+        logger.debug("Finished: exec_rebalancing_vm.")
+        return job_id
+
+    def exec_rebalancing_ct(self, proxmox_api: any, proxlb_data: Dict[str, Any], guest_name: str) -> None:
+        """
+        Executes the rebalancing of a container (CT) to a new node within the cluster.
+        This function initiates the migration of a specified CT to a target node as part of the
+        load balancing process. It logs the migration process and handles any exceptions that
+        may occur during the migration.
+        Args:
+            proxmox_api (object): The Proxmox API client instance used to interact with the Proxmox cluster.
+            proxlb_data (dict): A dictionary containing data related to the ProxLB load balancing configuration.
+            guest_name (str): The name of the guest CT to be migrated.
+        Raises:
+            proxmox_api.core.ResourceException: If an error occurs during the migration process.
+        Returns:
+            None
+        """
+        logger.debug("Starting: exec_rebalancing_ct.")
+        guest_id = proxlb_data["guests"][guest_name]["id"]
+        guest_node_current = proxlb_data["guests"][guest_name]["node_current"]
+        guest_node_target = proxlb_data["guests"][guest_name]["node_target"]
+        job_id = None
+
+        try:
+            logger.info(f"Balancing: Starting to migrate CT guest {guest_name} from {guest_node_current} to {guest_node_target}.")
+            job_id = proxmox_api.nodes(guest_node_current).lxc(guest_id).migrate().post(target=guest_node_target, restart=1)
+        except proxmoxer.core.ResourceException as proxmox_api_error:
+            logger.critical(f"Balancing: Failed to migrate guest {guest_name} of type CT due to some Proxmox errors. Please check if resource is locked or similar.")
+            logger.debug(f"Balancing: Failed to migrate guest {guest_name} of type CT due to some Proxmox errors: {proxmox_api_error}")
+
+        logger.debug("Finished: exec_rebalancing_ct.")
+        return job_id
+
+    def get_rebalancing_job_status(self, proxmox_api: any, proxlb_data: Dict[str, Any], guest_name: str, guest_current_node: str, job_id: int, retry_counter: int = 1) -> bool:
+        """
+        Monitors the status of a rebalancing job on a Proxmox node until it completes or a timeout is reached.
+
+        Args:
+            proxmox_api (object): The Proxmox API client instance.
+            proxlb_data (dict): The ProxLB configuration data.
+            guest_name (str): The name of the guest (virtual machine) being rebalanced.
+            guest_current_node (str): The current node where the guest is running.
+            job_id (str): The ID of the rebalancing job to monitor.
+            retry_counter (int, optional): The current retry count. Defaults to 1.
+
+        Returns:
+            bool: True if the job completed successfully, False otherwise.
+        """
+        logger.debug("Starting: get_rebalancing_job_status.")
+        job = proxmox_api.nodes(guest_current_node).tasks(job_id).status().get()
+
+        # Fetch actual migration job status if this got spawned by a HA job
+        if job["type"] == "hamigrate":
+            logger.debug(f"Balancing: Job ID {job_id} (guest: {guest_name}) is a HA migration job. Fetching underlying migration job...")
+            time.sleep(1)
+            vm_id = int(job["id"])
+            qm_migrate_jobs = proxmox_api.nodes(guest_current_node).tasks.get(typefilter="qmigrate", vmid=vm_id, start=0, source="active", limit=1)
+
+            if len(qm_migrate_jobs) > 0:
+                job = qm_migrate_jobs[0]
+                job_id = job["upid"]
+                logger.debug(f'Overwriting job polling for: ID {job_id} (guest: {guest_name}) by {job}')
+        else:
+            logger.debug(f"Balancing: Job ID {job_id} (guest: {guest_name}) is a standard migration job. Proceeding with status check.")
+
+        # Watch job id until it finalizes
+        # Note: Unsaved jobs are delivered in uppercase from Proxmox API
+        if job.get("status", "").lower() == "running":
+            # Do not hammer the API while
+            # watching the job status
+            time.sleep(10)
+            retry_counter += 1
+
+            # Run recursion until we hit the soft-limit of maximum migration time for a guest
+            if retry_counter < proxlb_data["meta"]["balancing"].get("max_job_validation", 1800):
+                logger.debug(f"Balancing: Job ID {job_id} (guest: {guest_name}) for migration is still running... (Run: {retry_counter})")
+                self.get_rebalancing_job_status(proxmox_api, proxlb_data, guest_name, guest_current_node, job_id, retry_counter)
+            else:
+                logger.warning(f"Balancing: Job ID {job_id} (guest: {guest_name}) for migration took too long. Please check manually.")
+                logger.debug("Finished: get_rebalancing_job_status.")
+                return False
+
+        # Validate job output for errors when finished
+        if job["status"] == "stopped":
+
+            if job["exitstatus"] == "OK":
+                logger.debug(f"Balancing: Job ID {job_id} (guest: {guest_name}) was successfully.")
+                logger.debug("Finished: get_rebalancing_job_status.")
+                return True
+            else:
+                logger.critical(f"Balancing: Job ID {job_id} (guest: {guest_name}) went into an error! Please check manually.")
+                logger.debug("Finished: get_rebalancing_job_status.")
+                return False