osism 0.20250605.0__py3-none-any.whl → 0.20250616.0__py3-none-any.whl

osism/tasks/conductor/ironic.py

@@ -1,15 +1,16 @@
 # SPDX-License-Identifier: Apache-2.0
 
-import ipaddress
 import json
 
 import jinja2
-from loguru import logger
 from pottery import Redlock
 
 from osism import utils as osism_utils
 from osism.tasks import netbox, openstack
-from osism.tasks.conductor.netbox import get_nb_device_query_list
+from osism.tasks.conductor.netbox import (
+    get_device_oob_ip,
+    get_nb_device_query_list_ironic,
+)
 from osism.tasks.conductor.utils import (
     deep_compare,
     deep_decrypt,
@@ -33,25 +34,32 @@ driver_params = {
 }
 
 
-def sync_ironic(get_ironic_parameters, force_update=False):
+def sync_ironic(request_id, get_ironic_parameters, force_update=False):
+    osism_utils.push_task_output(
+        request_id,
+        "Starting NetBox device synchronisation with ironic\n",
+    )
     devices = set()
-    nb_device_query_list = get_nb_device_query_list()
+    nb_device_query_list = get_nb_device_query_list_ironic()
     for nb_device_query in nb_device_query_list:
         devices |= set(netbox.get_devices(**nb_device_query))
 
-    # NOTE: Find nodes in Ironic which are no longer present in netbox and remove them
+    # NOTE: Find nodes in Ironic which are no longer present in NetBox and remove them
     device_names = {dev.name for dev in devices}
     nodes = openstack.baremetal_node_list()
     for node in nodes:
-        logger.info(f"Looking for {node['Name']} in netbox")
+        osism_utils.push_task_output(
+            request_id, f"Looking for {node['Name']} in NetBox\n"
+        )
         if node["Name"] not in device_names:
             if (
                 not node["Instance UUID"]
                 and node["Provisioning State"] in ["enroll", "manageable", "available"]
                 and node["Power State"] in ["power off", None]
             ):
-                logger.info(
-                    f"Cleaning up baremetal node not found in netbox: {node['Name']}"
+                osism_utils.push_task_output(
+                    request_id,
+                    f"Cleaning up baremetal node not found in NetBox: {node['Name']}\n",
                 )
                 for port in openstack.baremetal_port_list(
                     details=False, attributes=dict(node_uuid=node["UUID"])
@@ -59,14 +67,15 @@ def sync_ironic(get_ironic_parameters, force_update=False):
                     openstack.baremetal_port_delete(port.id)
                 openstack.baremetal_node_delete(node["UUID"])
             else:
-                logger.error(
+                osism_utils.push_task_output(
                     f"Cannot remove baremetal node because it is still provisioned or running: {node}"
                 )
 
-    # NOTE: Find nodes in netbox which are not present in Ironic and add them
+    # NOTE: Find nodes in NetBox which are not present in Ironic and add them
     for device in devices:
-        logger.info(f"Looking for {device.name} in ironic")
-        logger.info(device)
+        osism_utils.push_task_output(
+            request_id, f"Looking for {device.name} in ironic\n"
+        )
 
         node_interfaces = list(netbox.get_interfaces_by_device(device.name))
 
@@ -75,7 +84,7 @@ def sync_ironic(get_ironic_parameters, force_update=False):
             "ironic_parameters" in device.custom_fields
             and device.custom_fields["ironic_parameters"]
         ):
-            # NOTE: Update node attributes with overrides from netbox device
+            # NOTE: Update node attributes with overrides from NetBox device
             deep_merge(node_attributes, device.custom_fields["ironic_parameters"])
 
         # NOTE: Decrypt ansible vaulted secrets
@@ -132,32 +141,19 @@ def sync_ironic(get_ironic_parameters, force_update=False):
                 # NOTE: Render driver address field
                 address_key = driver_params[node_attributes["driver"]]["address"]
                 if address_key in node_attributes["driver_info"]:
-                    if device.oob_ip and "address" in device.oob_ip:
-                        node_mgmt_address = device.oob_ip["address"]
-                    else:
-                        node_mgmt_addresses = [
-                            interface["address"]
-                            for interface in node_interfaces
-                            if interface.mgmt_only
-                            and "address" in interface
-                            and interface["address"]
-                        ]
-                        if len(node_mgmt_addresses) > 0:
-                            node_mgmt_address = node_mgmt_addresses[0]
-                        else:
-                            node_mgmt_address = None
-                    if node_mgmt_address:
+                    oob_ip_result = get_device_oob_ip(device)
+                    if oob_ip_result:
+                        oob_ip, _ = (
+                            oob_ip_result  # Extract IP address, ignore prefix length
+                        )
                         node_attributes["driver_info"][address_key] = (
                             jinja2.Environment(loader=jinja2.BaseLoader())
                             .from_string(node_attributes["driver_info"][address_key])
-                            .render(
-                                remote_board_address=str(
-                                    ipaddress.ip_interface(node_mgmt_address).ip
-                                )
-                            )
+                            .render(remote_board_address=oob_ip)
                         )
         node_attributes.update({"resource_class": device.name})
-        # NOTE: Write metadata used for provisioning into 'extra' field, so that it is available during node deploy without querying the netbox again
+        # NOTE: Write metadata used for provisioning into 'extra' field, so that
+        #       it is available during node deploy without querying the NetBox again
         if "extra" not in node_attributes:
             node_attributes["extra"] = {}
         if (
@@ -191,10 +187,14 @@ def sync_ironic(get_ironic_parameters, force_update=False):
         )
         if lock.acquire(timeout=120):
             try:
-                logger.info(f"Processing device {device.name}")
+                osism_utils.push_task_output(
+                    request_id, f"Processing device {device.name}\n"
+                )
                 node = openstack.baremetal_node_show(device.name, ignore_missing=True)
                 if not node:
-                    logger.info(f"Creating baremetal node for {device.name}")
+                    osism_utils.push_task_output(
+                        request_id, f"Creating baremetal node for {device.name}\n"
+                    )
                     node = openstack.baremetal_node_create(device.name, node_attributes)
                 else:
                     # NOTE: The listener service only reacts to changes in the baremetal node. Explicitly sync provision and power state in case updates were missed by the listener.
@@ -218,8 +218,9 @@ def sync_ironic(get_ironic_parameters, force_update=False):
                             if not node_updates["driver_info"]:
                                 node_updates.pop("driver_info", None)
                     if node_updates or force_update:
-                        logger.info(
-                            f"Updating baremetal node for {device.name} with {node_updates}"
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Updating baremetal node for {device.name} with {node_updates}\n",
                         )
                         # NOTE: Do the actual updates with all values in node_attributes. Otherwise nested dicts like e.g. driver_info will be overwritten as a whole and contain only changed values
                         node = openstack.baremetal_node_update(
@@ -240,27 +241,31 @@ def sync_ironic(get_ironic_parameters, force_update=False):
                             == port["address"].upper()
                         ]
                         if not port:
-                            logger.info(
-                                f"Creating baremetal port with MAC address {port_attributes['address']} for {device.name}"
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Creating baremetal port with MAC address {port_attributes['address']} for {device.name}\n",
                             )
                             openstack.baremetal_port_create(port_attributes)
                         else:
                             node_ports.remove(port[0])
                 for node_port in node_ports:
-                    # NOTE: Delete remaining ports not found in netbox
-                    logger.info(
-                        f"Deleting baremetal port with MAC address {node_port['address']} for {device.name}"
+                    # NOTE: Delete remaining ports not found in NetBox
+                    osism_utils.push_task_output(
+                        request_id,
+                        f"Deleting baremetal port with MAC address {node_port['address']} for {device.name}\n",
                     )
                     openstack.baremetal_port_delete(node_port["id"])
 
                 node_validation = openstack.baremetal_node_validate(node["uuid"])
                 if node_validation["management"].result:
-                    logger.info(
-                        f"Validation of management interface successful for baremetal node for {device.name}"
+                    osism_utils.push_task_output(
+                        request_id,
+                        f"Validation of management interface successful for baremetal node for {device.name}\n",
                     )
                     if node["provision_state"] == "enroll":
-                        logger.info(
-                            f"Transitioning baremetal node to manageable state for {device.name}"
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Transitioning baremetal node to manageable state for {device.name}\n",
                         )
                         node = openstack.baremetal_node_set_provision_state(
                             node["uuid"], "manage"
@@ -268,14 +273,19 @@ def sync_ironic(get_ironic_parameters, force_update=False):
                         node = openstack.baremetal_node_wait_for_nodes_provision_state(
                             node["uuid"], "manageable"
                         )
-                        logger.info(f"Baremetal node for {device.name} is manageable")
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Baremetal node for {device.name} is manageable\n",
+                        )
                     if node_validation["boot"].result:
-                        logger.info(
-                            f"Validation of boot interface successful for baremetal node for {device.name}"
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Validation of boot interface successful for baremetal node for {device.name}\n",
                         )
                         if node["provision_state"] == "manageable":
-                            logger.info(
-                                f"Transitioning baremetal node to available state for {device.name}"
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Transitioning baremetal node to available state for {device.name}\n",
                             )
                             node = openstack.baremetal_node_set_provision_state(
                                 node["uuid"], "provide"
@@ -285,17 +295,20 @@ def sync_ironic(get_ironic_parameters, force_update=False):
                                     node["uuid"], "available"
                                 )
                             )
-                            logger.info(
-                                f"Baremetal node for {device.name} is available"
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Baremetal node for {device.name} is available\n",
                             )
                     else:
-                        logger.info(
-                            f"Validation of boot interface failed for baremetal node for {device.name}\nReason: {node_validation['boot'].reason}"
+                        osism_utils.push_task_output(
+                            request_id,
+                            f"Validation of boot interface failed for baremetal node for {device.name}\nReason: {node_validation['boot'].reason}\n",
                         )
                         if node["provision_state"] == "available":
                             # NOTE: Demote node to manageable
-                            logger.info(
-                                f"Transitioning baremetal node to manageable state for {device.name}"
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Transitioning baremetal node to manageable state for {device.name}\n",
                             )
                             node = openstack.baremetal_node_set_provision_state(
                                 node["uuid"], "manage"
@@ -305,19 +318,26 @@ def sync_ironic(get_ironic_parameters, force_update=False):
                                     node["uuid"], "manageable"
                                 )
                             )
-                            logger.info(
-                                f"Baremetal node for {device.name} is manageable"
+                            osism_utils.push_task_output(
+                                request_id,
+                                f"Baremetal node for {device.name} is manageable\n",
                             )
                 else:
-                    logger.info(
-                        f"Validation of management interface failed for baremetal node for {device.name}\nReason: {node_validation['management'].reason}"
+                    osism_utils.push_task_output(
+                        request_id,
+                        f"Validation of management interface failed for baremetal node for {device.name}\nReason: {node_validation['management'].reason}\n",
                     )
             except Exception as exc:
-                logger.info(
-                    f"Could not fully synchronize device {device.name} with ironic: {exc}"
+                osism_utils.push_task_output(
+                    request_id,
+                    f"Could not fully synchronize device {device.name} with ironic: {exc}\n",
                 )
             finally:
                 lock.release()
 
         else:
-            logger.error("Could not acquire lock for node {device.name}")
+            osism_utils.push_task_output(
+                "Could not acquire lock for node {device.name}"
+            )
+
+    osism_utils.finish_task_output(request_id, rc=0)

osism/tasks/conductor/netbox.py

@@ -3,11 +3,11 @@
 from loguru import logger
 import yaml
 
-from osism import settings
+from osism import settings, utils
 from osism.tasks import netbox
 
 
-def get_nb_device_query_list():
+def get_nb_device_query_list_ironic():
     try:
         supported_nb_device_filters = [
             "site",
@@ -18,7 +18,7 @@ def get_nb_device_query_list():
             "tag",
             "state",
         ]
-        nb_device_query_list = yaml.safe_load(settings.NETBOX_FILTER_CONDUCTOR)
+        nb_device_query_list = yaml.safe_load(settings.NETBOX_FILTER_CONDUCTOR_IRONIC)
         if type(nb_device_query_list) is not list:
             raise TypeError
         for nb_device_query in nb_device_query_list:
@@ -27,7 +27,7 @@ def get_nb_device_query_list():
             for key in list(nb_device_query.keys()):
                 if key not in supported_nb_device_filters:
                     raise ValueError
-                # NOTE: Only "location_id" and "rack_id" are supported by netbox
+                # NOTE: Only "location_id" and "rack_id" are supported by NetBox
                 if key in ["location", "rack"]:
                     value_name = nb_device_query.pop(key, "")
                     if key == "location":
@@ -40,11 +40,272 @@ def get_nb_device_query_list():
                         raise ValueError(f"Invalid name {value_name} for {key}")
     except (yaml.YAMLError, TypeError):
         logger.error(
-            f"Setting NETBOX_FILTER_CONDUCTOR needs to be an array of mappings containing supported netbox device filters: {supported_nb_device_filters}"
+            f"Setting NETBOX_FILTER_CONDUCTOR_IRONIC needs to be an array of mappings containing supported NetBox device filters: {supported_nb_device_filters}"
         )
         nb_device_query_list = []
     except ValueError as exc:
-        logger.error(f"Unknown value in NETBOX_FILTER_CONDUCTOR: {exc}")
+        logger.error(f"Unknown value in NETBOX_FILTER_CONDUCTOR_IRONIC: {exc}")
         nb_device_query_list = []
 
     return nb_device_query_list
+
+
+def get_nb_device_query_list_sonic():
+    try:
+        supported_nb_device_filters = [
+            "site",
+            "region",
+            "site_group",
+            "location",
+            "rack",
+            "tag",
+            "state",
+        ]
+        nb_device_query_list = yaml.safe_load(settings.NETBOX_FILTER_CONDUCTOR_SONIC)
+        if type(nb_device_query_list) is not list:
+            raise TypeError
+        for nb_device_query in nb_device_query_list:
+            if type(nb_device_query) is not dict:
+                raise TypeError
+            for key in list(nb_device_query.keys()):
+                if key not in supported_nb_device_filters:
+                    raise ValueError
+                # NOTE: Only "location_id" and "rack_id" are supported by NetBox
+                if key in ["location", "rack"]:
+                    value_name = nb_device_query.pop(key, "")
+                    if key == "location":
+                        value_id = netbox.get_location_id(value_name)
+                    elif key == "rack":
+                        value_id = netbox.get_rack_id(value_name)
+                    if value_id:
+                        nb_device_query.update({key + "_id": value_id})
+                    else:
+                        raise ValueError(f"Invalid name {value_name} for {key}")
+    except (yaml.YAMLError, TypeError):
+        logger.error(
+            f"Setting NETBOX_FILTER_CONDUCTOR_SONIC needs to be an array of mappings containing supported NetBox device filters: {supported_nb_device_filters}"
+        )
+        nb_device_query_list = []
+    except ValueError as exc:
+        logger.error(f"Unknown value in NETBOX_FILTER_CONDUCTOR_SONIC: {exc}")
+        nb_device_query_list = []
+
+    return nb_device_query_list
+
+
+def get_device_oob_ip(device):
+    """Get out-of-band IP address for device management interface.
+
+    Args:
+        device: NetBox device object
+
+    Returns:
+        tuple: (IP address, prefix length) for management interface or None
+               Example: ('192.168.1.10', 24)
+    """
+    import ipaddress
+
+    try:
+        oob_ip_with_prefix = None
+
+        # First check if device has oob_ip field set
+        if hasattr(device, "oob_ip") and device.oob_ip:
+            oob_ip_with_prefix = device.oob_ip
+        else:
+            # Fall back to management interfaces
+            interfaces = utils.nb.dcim.interfaces.filter(device_id=device.id)
+
+            for interface in interfaces:
+                if interface.mgmt_only:
+                    # Get IP addresses assigned to this interface
+                    ip_addresses = utils.nb.ipam.ip_addresses.filter(
+                        assigned_object_id=interface.id,
+                    )
+
+                    for ip_addr in ip_addresses:
+                        if ip_addr.address:
+                            oob_ip_with_prefix = ip_addr.address
+                            break
+                    if oob_ip_with_prefix:
+                        break
+
+        if oob_ip_with_prefix:
+            # Parse the IP address with prefix (e.g., "192.168.1.10/24")
+            ip_interface = ipaddress.ip_interface(oob_ip_with_prefix)
+            ip_address = str(ip_interface.ip)
+            prefix_length = ip_interface.network.prefixlen
+
+            logger.debug(
+                f"Found OOB IP for device {device.name}: {ip_address}/{prefix_length}"
+            )
+
+            # Return tuple of (IP address, prefix length)
+            return (ip_address, prefix_length)
+
+    except Exception as e:
+        logger.warning(f"Could not get OOB IP for device {device.name}: {e}")
+
+    return None
+
+
+def get_device_vlans(device):
+    """Get VLANs configured on device interfaces.
+
+    Args:
+        device: NetBox device object
+
+    Returns:
+        dict: Dictionary with VLAN information
+              {
+                  'vlans': {vid: {'name': name, 'description': desc}},
+                  'vlan_members': {vid: {'port_name': 'tagging_mode'}},
+                  'vlan_interfaces': {vid: {'addresses': [ip_with_prefix, ...]}}
+              }
+    """
+    vlans = {}
+    vlan_members = {}
+    vlan_interfaces = {}
+
+    try:
+        # Get all interfaces for the device and convert to list for multiple iterations
+        interfaces = list(utils.nb.dcim.interfaces.filter(device_id=device.id))
+
+        for interface in interfaces:
+            # Skip management interfaces and virtual interfaces
+            if interface.mgmt_only or (
+                hasattr(interface, "type")
+                and interface.type
+                and interface.type.value == "virtual"
+            ):
+                continue
+
+            # Process untagged VLAN
+            if hasattr(interface, "untagged_vlan") and interface.untagged_vlan:
+                vlan = interface.untagged_vlan
+                vid = vlan.vid
+
+                # Add VLAN info if not already present
+                if vid not in vlans:
+                    vlans[vid] = {
+                        "name": vlan.name or f"Vlan{vid}",
+                        "description": vlan.description or "",
+                    }
+
+                # Add interface to VLAN members as untagged
+                if vid not in vlan_members:
+                    vlan_members[vid] = {}
+
+                # Use original NetBox interface name - conversion will be done in sonic.py
+                vlan_members[vid][interface.name] = "untagged"
+
+            # Process tagged VLANs
+            if hasattr(interface, "tagged_vlans") and interface.tagged_vlans:
+                for vlan in interface.tagged_vlans:
+                    vid = vlan.vid
+
+                    # Add VLAN info if not already present
+                    if vid not in vlans:
+                        vlans[vid] = {
+                            "name": vlan.name or f"Vlan{vid}",
+                            "description": vlan.description or "",
+                        }
+
+                    # Add interface to VLAN members as tagged
+                    if vid not in vlan_members:
+                        vlan_members[vid] = {}
+
+                    # Use original NetBox interface name - conversion will be done in sonic.py
+                    vlan_members[vid][interface.name] = "tagged"
+
+        # Get VLAN interfaces (SVIs) - virtual interfaces with VLAN assignments
+        for interface in interfaces:
+            # Check if interface is virtual type and has VLAN assignment
+            if (
+                hasattr(interface, "type")
+                and interface.type
+                and interface.type.value == "virtual"
+                and interface.name.startswith("Vlan")
+            ):
+                try:
+                    vid = int(interface.name[4:])
+                    # Get IP addresses for this VLAN interface
+                    ip_addresses = utils.nb.ipam.ip_addresses.filter(
+                        assigned_object_id=interface.id,
+                    )
+
+                    addresses = []
+                    for ip_addr in ip_addresses:
+                        if ip_addr.address:
+                            addresses.append(ip_addr.address)
+
+                    if addresses:
+                        if vid not in vlan_interfaces:
+                            vlan_interfaces[vid] = {}
+                        # Store all IP addresses for this VLAN interface
+                        vlan_interfaces[vid]["addresses"] = addresses
+                except (ValueError, IndexError):
+                    # Skip if interface name doesn't follow Vlan<number> pattern
+                    pass
+
+    except Exception as e:
+        logger.warning(f"Could not get VLANs for device {device.name}: {e}")
+
+    return {
+        "vlans": vlans,
+        "vlan_members": vlan_members,
+        "vlan_interfaces": vlan_interfaces,
+    }
+
+
+def get_device_loopbacks(device):
+    """Get Loopback interfaces configured on device.
+
+    Args:
+        device: NetBox device object
+
+    Returns:
+        dict: Dictionary with Loopback information
+              {
+                  'loopbacks': {'Loopback0': {'addresses': [ip_with_prefix, ...]}}
+              }
+    """
+    loopbacks = {}
+
+    try:
+        # Get all interfaces for the device
+        interfaces = list(utils.nb.dcim.interfaces.filter(device_id=device.id))
+
+        for interface in interfaces:
+            # Check if interface is virtual type and is a Loopback interface
+            if (
+                hasattr(interface, "type")
+                and interface.type
+                and interface.type.value == "virtual"
+                and interface.name.startswith("Loopback")
+            ):
+
+                try:
+                    # Get IP addresses for this Loopback interface
+                    ip_addresses = utils.nb.ipam.ip_addresses.filter(
+                        assigned_object_id=interface.id,
+                    )
+
+                    addresses = []
+                    for ip_addr in ip_addresses:
+                        if ip_addr.address:
+                            addresses.append(ip_addr.address)
+
+                    if addresses:
+                        loopbacks[interface.name] = {"addresses": addresses}
+
+                except Exception as e:
+                    logger.debug(
+                        f"Error processing Loopback interface {interface.name}: {e}"
+                    )
+
+    except Exception as e:
+        logger.warning(
+            f"Could not get Loopback interfaces for device {device.name}: {e}"
+        )
+
+    return {"loopbacks": loopbacks}