osism 0.20250602.0__py3-none-any.whl → 0.20250616.0__py3-none-any.whl

osism/tasks/conductor/__init__.py

@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import copy
+from celery import Celery
+from celery.signals import worker_process_init
+from loguru import logger
+
+from osism.tasks import Config
+from osism.tasks.conductor.config import get_configuration
+from osism.tasks.conductor.ironic import sync_ironic as _sync_ironic
+from osism.tasks.conductor.sonic import sync_sonic as _sync_sonic
+
+
+# App configuration
+app = Celery("conductor")
+app.config_from_object(Config)
+
+
+@worker_process_init.connect
+def celery_init_worker(**kwargs):
+    pass
+
+
+@app.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    pass
+
+
+# Tasks
+@app.task(bind=True, name="osism.tasks.conductor.get_ironic_parameters")
+def get_ironic_parameters(self):
+    configuration = get_configuration()
+    if "ironic_parameters" in configuration:
+        # NOTE: Do not pass by reference, everybody gets their own copy to work with
+        return copy.deepcopy(configuration["ironic_parameters"])
+
+    return {}
+
+
+@app.task(bind=True, name="osism.tasks.conductor.sync_netbox")
+def sync_netbox(self, force_update=False):
+    logger.info("Not implemented")
+
+
+@app.task(bind=True, name="osism.tasks.conductor.sync_ironic")
+def sync_ironic(self, force_update=False):
+    _sync_ironic(self.request.id, get_ironic_parameters, force_update)
+
+
+@app.task(bind=True, name="osism.tasks.conductor.sync_sonic")
+def sync_sonic(self):
+    return _sync_sonic()
+
+
+__all__ = [
+    "app",
+    "get_ironic_parameters",
+    "sync_netbox",
+    "sync_ironic",
+    "sync_sonic",
+]

osism/tasks/conductor/config.py

@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import uuid
+
+from loguru import logger
+import yaml
+
+from osism.tasks import Config, openstack
+
+
+def is_uuid(value):
+    """Check if a string is a valid UUID."""
+    try:
+        uuid.UUID(value)
+        return True
+    except (ValueError, AttributeError):
+        return False
+
+
+def get_configuration():
+    with open("/etc/conductor.yml") as fp:
+        configuration = yaml.load(fp, Loader=yaml.SafeLoader)
+
+        if not configuration:
+            logger.warning(
+                "The conductor configuration is empty. That's probably wrong"
+            )
+            return {}
+
+        if Config.enable_ironic.lower() not in ["true", "yes"]:
+            return configuration
+
+        if "ironic_parameters" not in configuration:
+            logger.error("ironic_parameters not found in the conductor configuration")
+            return configuration
+
+        if "instance_info" in configuration["ironic_parameters"]:
+            if "image_source" in configuration["ironic_parameters"]["instance_info"]:
+                image_source = configuration["ironic_parameters"]["instance_info"][
+                    "image_source"
+                ]
+                if not is_uuid(image_source):
+                    result = openstack.image_get(image_source)
+                    configuration["ironic_parameters"]["instance_info"][
+                        "image_source"
+                    ] = result.id
+
+        if "driver_info" in configuration["ironic_parameters"]:
+            if "deploy_kernel" in configuration["ironic_parameters"]["driver_info"]:
+                deploy_kernel = configuration["ironic_parameters"]["driver_info"][
+                    "deploy_kernel"
+                ]
+                if not is_uuid(deploy_kernel):
+                    result = openstack.image_get(deploy_kernel)
+                    configuration["ironic_parameters"]["driver_info"][
+                        "deploy_kernel"
+                    ] = result.id
+
+            if "deploy_ramdisk" in configuration["ironic_parameters"]["driver_info"]:
+                deploy_ramdisk = configuration["ironic_parameters"]["driver_info"][
+                    "deploy_ramdisk"
+                ]
+                if not is_uuid(deploy_ramdisk):
+                    result = openstack.image_get(deploy_ramdisk)
+                    configuration["ironic_parameters"]["driver_info"][
+                        "deploy_ramdisk"
+                    ] = result.id
+
+            if "cleaning_network" in configuration["ironic_parameters"]["driver_info"]:
+                result = openstack.network_get(
+                    configuration["ironic_parameters"]["driver_info"][
+                        "cleaning_network"
+                    ]
+                )
+                configuration["ironic_parameters"]["driver_info"][
+                    "cleaning_network"
+                ] = result.id
+
+            if (
+                "provisioning_network"
+                in configuration["ironic_parameters"]["driver_info"]
+            ):
+                result = openstack.network_get(
+                    configuration["ironic_parameters"]["driver_info"][
+                        "provisioning_network"
+                    ]
+                )
+                configuration["ironic_parameters"]["driver_info"][
+                    "provisioning_network"
+                ] = result.id
+
+        return configuration

osism/tasks/conductor/ironic.py

@@ -0,0 +1,343 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+
+import jinja2
+from pottery import Redlock
+
+from osism import utils as osism_utils
+from osism.tasks import netbox, openstack
+from osism.tasks.conductor.netbox import (
+    get_device_oob_ip,
+    get_nb_device_query_list_ironic,
+)
+from osism.tasks.conductor.utils import (
+    deep_compare,
+    deep_decrypt,
+    deep_merge,
+    get_vault,
+)
+
+
+driver_params = {
+    "ipmi": {
+        "address": "ipmi_address",
+        "port": "ipmi_port",
+        "password": "ipmi_password",
+        "username": "ipmi_username",
+    },
+    "redfish": {
+        "address": "redfish_address",
+        "password": "redfish_password",
+        "username": "redfish_username",
+    },
+}
+
+
+def sync_ironic(request_id, get_ironic_parameters, force_update=False):
+    osism_utils.push_task_output(
+        request_id,
+        "Starting NetBox device synchronisation with ironic\n",
+    )
+    devices = set()
+    nb_device_query_list = get_nb_device_query_list_ironic()
+    for nb_device_query in nb_device_query_list:
+        devices |= set(netbox.get_devices(**nb_device_query))
+
+    # NOTE: Find nodes in Ironic which are no longer present in NetBox and remove them
+    device_names = {dev.name for dev in devices}
+    nodes = openstack.baremetal_node_list()
+    for node in nodes:
+        osism_utils.push_task_output(
+            request_id, f"Looking for {node['Name']} in NetBox\n"
+        )
+        if node["Name"] not in device_names:
+            if (
+                not node["Instance UUID"]
+                and node["Provisioning State"] in ["enroll", "manageable", "available"]
+                and node["Power State"] in ["power off", None]
+            ):
+                osism_utils.push_task_output(
+                    request_id,
+                    f"Cleaning up baremetal node not found in NetBox: {node['Name']}\n",
+                )
+                for port in openstack.baremetal_port_list(
+                    details=False, attributes=dict(node_uuid=node["UUID"])
+                ):
+                    openstack.baremetal_port_delete(port.id)
+                openstack.baremetal_node_delete(node["UUID"])
+            else:
+                osism_utils.push_task_output(
+                    f"Cannot remove baremetal node because it is still provisioned or running: {node}"
+                )
+
+    # NOTE: Find nodes in NetBox which are not present in Ironic and add them
+    for device in devices:
+        osism_utils.push_task_output(
+            request_id, f"Looking for {device.name} in ironic\n"
+        )
+
+        node_interfaces = list(netbox.get_interfaces_by_device(device.name))
+
+        node_attributes = get_ironic_parameters()
+        if (
+            "ironic_parameters" in device.custom_fields
+            and device.custom_fields["ironic_parameters"]
+        ):
+            # NOTE: Update node attributes with overrides from NetBox device
+            deep_merge(node_attributes, device.custom_fields["ironic_parameters"])
+
+        # NOTE: Decrypt ansible vaulted secrets
+        vault = get_vault()
+        deep_decrypt(node_attributes, vault)
+
+        node_secrets = device.custom_fields.get("secrets", {})
+        if node_secrets is None:
+            node_secrets = {}
+        deep_decrypt(node_secrets, vault)
+
+        if (
+            "driver" in node_attributes
+            and node_attributes["driver"] in driver_params.keys()
+        ):
+            if "driver_info" in node_attributes:
+                # NOTE: Remove all fields belonging to a different driver
+                unused_drivers = [
+                    driver
+                    for driver in driver_params.keys()
+                    if driver != node_attributes["driver"]
+                ]
+                for key in list(node_attributes["driver_info"].keys()):
+                    for driver in unused_drivers:
+                        if key.startswith(driver + "_"):
+                            node_attributes["driver_info"].pop(key, None)
+
+                # NOTE: Render driver username field
+                username_key = driver_params[node_attributes["driver"]]["username"]
+                if username_key in node_attributes["driver_info"]:
+                    node_attributes["driver_info"][username_key] = (
+                        jinja2.Environment(loader=jinja2.BaseLoader())
+                        .from_string(node_attributes["driver_info"][username_key])
+                        .render(
+                            remote_board_username=str(
+                                node_secrets.get("remote_board_username", "admin")
+                            )
+                        )
+                    )
+
+                # NOTE: Render driver password field
+                password_key = driver_params[node_attributes["driver"]]["password"]
+                if password_key in node_attributes["driver_info"]:
+                    node_attributes["driver_info"][password_key] = (
+                        jinja2.Environment(loader=jinja2.BaseLoader())
+                        .from_string(node_attributes["driver_info"][password_key])
+                        .render(
+                            remote_board_password=str(
+                                node_secrets.get("remote_board_password", "password")
+                            )
+                        )
+                    )
+
+                # NOTE: Render driver address field
+                address_key = driver_params[node_attributes["driver"]]["address"]
+                if address_key in node_attributes["driver_info"]:
+                    oob_ip_result = get_device_oob_ip(device)
+                    if oob_ip_result:
+                        oob_ip, _ = (
+                            oob_ip_result  # Extract IP address, ignore prefix length
+                        )
+                        node_attributes["driver_info"][address_key] = (
+                            jinja2.Environment(loader=jinja2.BaseLoader())
+                            .from_string(node_attributes["driver_info"][address_key])
+                            .render(remote_board_address=oob_ip)
+                        )
+        node_attributes.update({"resource_class": device.name})
+        # NOTE: Write metadata used for provisioning into 'extra' field, so that
+        #       it is available during node deploy without querying the NetBox again
+        if "extra" not in node_attributes:
+            node_attributes["extra"] = {}
+        if (
+            "netplan_parameters" in device.custom_fields
+            and device.custom_fields["netplan_parameters"]
+        ):
+            node_attributes["extra"].update(
+                {
+                    "netplan_parameters": json.dumps(
+                        device.custom_fields["netplan_parameters"]
+                    )
+                }
+            )
+        if (
+            "frr_parameters" in device.custom_fields
+            and device.custom_fields["frr_parameters"]
+        ):
+            node_attributes["extra"].update(
+                {"frr_parameters": json.dumps(device.custom_fields["frr_parameters"])}
+            )
+        ports_attributes = [
+            dict(address=interface.mac_address)
+            for interface in node_interfaces
+            if interface.enabled and not interface.mgmt_only and interface.mac_address
+        ]
+
+        lock = Redlock(
+            key=f"lock_osism_tasks_conductor_sync_ironic-{device.name}",
+            masters={osism_utils.redis},
+            auto_release_time=600,
+        )
+        if lock.acquire(timeout=120):
+            try:
+                osism_utils.push_task_output(
+                    request_id, f"Processing device {device.name}\n"
+                )
+                node = openstack.baremetal_node_show(device.name, ignore_missing=True)
+                if not node:
+                    osism_utils.push_task_output(
+                        request_id, f"Creating baremetal node for {device.name}\n"
+                    )
+                    node = openstack.baremetal_node_create(device.name, node_attributes)
+                else:
+                    # NOTE: The listener service only reacts to changes in the baremetal node. Explicitly sync provision and power state in case updates were missed by the listener.
+                    if (
+                        device.custom_fields["provision_state"]
+                        != node["provision_state"]
+                    ):
+                        netbox.set_provision_state(device.name, node["provision_state"])
+                    if device.custom_fields["power_state"] != node["power_state"]:
+                        netbox.set_power_state(device.name, node["power_state"])
+                    # NOTE: Check whether the baremetal node needs to be updated
+                    node_updates = {}
+                    deep_compare(node_attributes, node, node_updates)
+                    if "driver_info" in node_updates:
+                        # NOTE: The password is not returned by ironic, so we cannot make a comparision and it would always be updated. Therefore we pop it from the dictionary
+                        password_key = driver_params[node_attributes["driver"]][
+                            "password"
+                        ]
+                        if password_key in node_updates["driver_info"]:
+                            node_updates["driver_info"].pop(password_key, None)
+                            if not node_updates["driver_info"]:
+                                node_updates.pop("driver_info", None)
+                    if node_updates or force_update:
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Updating baremetal node for {device.name} with {node_updates}\n",
+                        )
+                        # NOTE: Do the actual updates with all values in node_attributes. Otherwise nested dicts like e.g. driver_info will be overwritten as a whole and contain only changed values
+                        node = openstack.baremetal_node_update(
+                            node["uuid"], node_attributes
+                        )
+
+                node_ports = openstack.baremetal_port_list(
+                    details=False, attributes=dict(node_uuid=node["uuid"])
+                )
+                # NOTE: Baremetal ports are only required for (i)pxe boot
+                if node["boot_interface"] in ["pxe", "ipxe"]:
+                    for port_attributes in ports_attributes:
+                        port_attributes.update({"node_id": node["uuid"]})
+                        port = [
+                            port
+                            for port in node_ports
+                            if port_attributes["address"].upper()
+                            == port["address"].upper()
+                        ]
+                        if not port:
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Creating baremetal port with MAC address {port_attributes['address']} for {device.name}\n",
+                            )
+                            openstack.baremetal_port_create(port_attributes)
+                        else:
+                            node_ports.remove(port[0])
+                for node_port in node_ports:
+                    # NOTE: Delete remaining ports not found in NetBox
+                    osism_utils.push_task_output(
+                        request_id,
+                        f"Deleting baremetal port with MAC address {node_port['address']} for {device.name}\n",
+                    )
+                    openstack.baremetal_port_delete(node_port["id"])
+
+                node_validation = openstack.baremetal_node_validate(node["uuid"])
+                if node_validation["management"].result:
+                    osism_utils.push_task_output(
+                        request_id,
+                        f"Validation of management interface successful for baremetal node for {device.name}\n",
+                    )
+                    if node["provision_state"] == "enroll":
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Transitioning baremetal node to manageable state for {device.name}\n",
+                        )
+                        node = openstack.baremetal_node_set_provision_state(
+                            node["uuid"], "manage"
+                        )
+                        node = openstack.baremetal_node_wait_for_nodes_provision_state(
+                            node["uuid"], "manageable"
+                        )
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Baremetal node for {device.name} is manageable\n",
+                        )
+                    if node_validation["boot"].result:
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Validation of boot interface successful for baremetal node for {device.name}\n",
+                        )
+                        if node["provision_state"] == "manageable":
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Transitioning baremetal node to available state for {device.name}\n",
+                            )
+                            node = openstack.baremetal_node_set_provision_state(
+                                node["uuid"], "provide"
+                            )
+                            node = (
+                                openstack.baremetal_node_wait_for_nodes_provision_state(
+                                    node["uuid"], "available"
+                                )
+                            )
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Baremetal node for {device.name} is available\n",
+                            )
+                    else:
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Validation of boot interface failed for baremetal node for {device.name}\nReason: {node_validation['boot'].reason}\n",
+                        )
+                        if node["provision_state"] == "available":
+                            # NOTE: Demote node to manageable
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Transitioning baremetal node to manageable state for {device.name}\n",
+                            )
+                            node = openstack.baremetal_node_set_provision_state(
+                                node["uuid"], "manage"
+                            )
+                            node = (
+                                openstack.baremetal_node_wait_for_nodes_provision_state(
+                                    node["uuid"], "manageable"
+                                )
+                            )
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Baremetal node for {device.name} is manageable\n",
+                            )
+                else:
+                    osism_utils.push_task_output(
+                        request_id,
+                        f"Validation of management interface failed for baremetal node for {device.name}\nReason: {node_validation['management'].reason}\n",
+                    )
+            except Exception as exc:
+                osism_utils.push_task_output(
+                    request_id,
+                    f"Could not fully synchronize device {device.name} with ironic: {exc}\n",
+                )
+            finally:
+                lock.release()
+
+        else:
+            osism_utils.push_task_output(
+                "Could not acquire lock for node {device.name}"
+            )
+
+    osism_utils.finish_task_output(request_id, rc=0)