osism 0.20250331.0__py3-none-any.whl → 0.20250425.0__py3-none-any.whl

osism/services/listener.py

@@ -11,8 +11,7 @@ from loguru import logger
 import json
 import requests
 
-from osism import utils
-from osism.tasks import netbox, openstack
+from osism.tasks import netbox
 from osism import settings
 
 EXCHANGE_NAME = "ironic"
@@ -40,16 +39,13 @@ class BaremetalEvents:
                     },
                     "maintenance_set": {"end": self.node_maintenance_set_end},
                     "provision_set": {
+                        "start": self.node_provision_set_start,
                         "end": self.node_provision_set_end,
                         "success": self.node_provision_set_success,
                     },
                     "delete": {"end": self.node_delete_end},
                     "create": {"end": self.node_create_end},
                 },
-                "port": {
-                    "create": {"end": self.port_create_end},
-                    "update": {"end": self.port_update_end},
-                },
             }
         }
 
@@ -78,7 +74,7 @@ class BaremetalEvents:
         logger.info(
             f"baremetal.node.power_set.end ## {name} ## {object_data['power_state']}"
         )
-        netbox.set_state.delay(name, object_data["power_state"], "power")
+        netbox.set_power_state.delay(name, object_data["power_state"])
 
     def node_power_state_corrected_success(self, payload: dict[Any, Any]) -> None:
         object_data = self.get_object_data(payload)
@@ -86,7 +82,7 @@ class BaremetalEvents:
         logger.info(
             f"baremetal.node.power_state_corrected.success ## {name} ## {object_data['power_state']}"
         )
-        netbox.set_state.delay(name, object_data["power_state"], "power")
+        netbox.set_power_state.delay(name, object_data["power_state"])
 
     def node_maintenance_set_end(self, payload: dict[Any, Any]) -> None:
         object_data = self.get_object_data(payload)
@@ -94,7 +90,7 @@ class BaremetalEvents:
         logger.info(
             f"baremetal.node.maintenance_set.end ## {name} ## {object_data['maintenance']}"
         )
-        netbox.set_maintenance.delay(name, object_data["maintenance"])
+        netbox.set_maintenance.delay(name, state=object_data["maintenance"])
 
     def node_provision_set_success(self, payload: dict[Any, Any]) -> None:
         # A provision status was successfully set, update it in the netbox
@@ -103,80 +99,37 @@ class BaremetalEvents:
         logger.info(
             f"baremetal.node.provision_set.success ## {name} ## {object_data['provision_state']}"
         )
-        netbox.set_state.delay(name, object_data["provision_state"], "provision")
+        netbox.set_provision_state.delay(name, object_data["provision_state"])
 
-    def node_provision_set_end(self, payload: dict[Any, Any]) -> None:
+    def node_provision_set_start(self, payload: dict[Any, Any]) -> None:
         object_data = self.get_object_data(payload)
         name = object_data["name"]
         logger.info(
-            f"baremetal.node.provision_set.end ## {name} ## {object_data['provision_state']}"
-        )
-        netbox.set_state.delay(name, object_data["provision_state"], "provision")
-
-        if (
-            object_data["previous_provision_state"] == "inspect wait"
-            and object_data["event"] == "done"
-        ):
-            netbox.set_state.delay(name, "introspected", "introspection")
-            openstack.baremetal_set_node_provision_state.delay(name, "provide")
-
-    def port_create_end(self, payload: dict[Any, Any]) -> None:
-        object_data = self.get_object_data(payload)
-        name = object_data["name"]
-        logger.info(f"baremetal.port.create.end ## {object_data['uuid']}")
-
-        mac_address = object_data["address"]
-        interface_a = utils.nb.dcim.interfaces.get(mac_address=mac_address)
-        device_a = interface_a.device
-
-        task = openstack.baremetal_get_network_interface_name.delay(
-            device_a.name, mac_address
-        )
-        task.wait(timeout=None, interval=0.5)
-        network_interface_name = task.get()
-
-        netbox.update_network_interface_name.delay(
-            object_data["address"], network_interface_name
+            f"baremetal.node.provision_set.start ## {name} ## {object_data['provision_state']}"
         )
+        netbox.set_provision_state.delay(name, object_data["provision_state"])
 
-    def port_update_end(self, payload: dict[Any, Any]) -> None:
+    def node_provision_set_end(self, payload: dict[Any, Any]) -> None:
         object_data = self.get_object_data(payload)
         name = object_data["name"]
-        logger.info(f"baremetal.port.update.end ## {object_data['uuid']}")
-
-        mac_address = object_data["address"]
-        interface_a = utils.nb.dcim.interfaces.get(mac_address=mac_address)
-        device_a = interface_a.device
-
-        task = openstack.baremetal_get_network_interface_name.delay(
-            device_a.name, mac_address
-        )
-        task.wait(timeout=None, interval=0.5)
-        network_interface_name = task.get()
-
-        netbox.update_network_interface_name.delay(
-            object_data["address"], network_interface_name
+        logger.info(
+            f"baremetal.node.provision_set.end ## {name} ## {object_data['provision_state']}"
         )
+        netbox.set_provision_state.delay(name, object_data["provision_state"])
 
     def node_delete_end(self, payload: dict[Any, Any]) -> None:
         object_data = self.get_object_data(payload)
         name = object_data["name"]
         logger.info(f"baremetal.node.delete.end ## {name}")
-
-        netbox.set_state.delay(name, "unregistered", "ironic")
-        netbox.set_state.delay(name, None, "provision")
-        netbox.set_state.delay(name, None, "power")
-        netbox.set_state.delay(name, None, "introspection")
-        netbox.set_state.delay(name, None, "deployment")
-
-        # remove internal flavor
-        openstack.baremetal_delete_internal_flavor.delay(name)
+        netbox.set_provision_state.delay(name, None)
+        netbox.set_power_state.delay(name, None)
 
     def node_create_end(self, payload: dict[Any, Any]) -> None:
         object_data = self.get_object_data(payload)
         name = object_data["name"]
         logger.info(f"baremetal.node.create.end ## {name}")
-        netbox.set_state.delay(name, "registered", "ironic")
+        netbox.set_provision_state.delay(name, object_data["provision_state"])
+        netbox.set_power_state.delay(name, object_data["power_state"])
 
 
 class NotificationsDump(ConsumerMixin):
@@ -207,7 +160,17 @@ class NotificationsDump(ConsumerMixin):
 
     def on_message(self, body, message):
         data = json.loads(body["oslo.message"])
-        # logger.info(data)
+        logger.debug(
+            data["event_type"]
+            + ": "
+            + str(
+                {
+                    k: v
+                    for k, v in data["payload"]["ironic_object.data"].items()
+                    if k in ["provision_state", "power_state"]
+                }
+            )
+        )
 
         if self.osism_api_session:
             tries = 1
@@ -268,8 +231,6 @@ class NotificationsDump(ConsumerMixin):
             handler = self.baremetal_events.get_handler(data["event_type"])
             handler(data["payload"])
 
-        logger.info(self.baremetal_events.get_object_data(data["payload"]))
-
 
 def main():
     while True:

osism/settings.py

@@ -34,3 +34,12 @@ INVENTORY_RECONCILER_SCHEDULE = float(
 )
 
 OSISM_API_URL = os.getenv("OSISM_API_URL", None)
+
+OSISM_CONDUCTOR_NETBOX_FILTER_LIST = os.getenv(
+    "OSISM_CONDUCTOR_NETBOX_FILTER_LIST",
+    "[{'state': 'active', 'tag': ['managed-by-ironic']}]",
+)
+
+NETBOX_SECONDARIES = (
+    os.getenv("NETBOX_SECONDARIES", read_secret("NETBOX_SECONDARIES")) or "[]"
+)

osism/tasks/__init__.py

@@ -183,11 +183,16 @@ def run_command(
     *arguments,
     publish=True,
     locking=False,
+    ignore_env=False,
     auto_release_time=3600,
 ):
     result = ""
-    command_env = os.environ.copy()
-    command_env.update(env)
+
+    if ignore_env:
+        command_env = env
+    else:
+        command_env = os.environ.copy()
+        command_env.update(env)
 
     if locking:
         lock = Redlock(

osism/tasks/conductor.py

@@ -2,30 +2,29 @@
 
 from celery import Celery
 from celery.signals import worker_process_init
-import keystoneauth1
+import copy
+import ipaddress
+import jinja2
 from loguru import logger
-import openstack
+from pottery import Redlock
 import yaml
 
-from osism.tasks import Config
+from osism import settings
+from osism import utils
+from osism.tasks import Config, netbox, openstack
 
 app = Celery("conductor")
 app.config_from_object(Config)
 
 
 configuration = {}
+nb_device_query_list = None
 
 
 @worker_process_init.connect
 def celery_init_worker(**kwargs):
     global configuration
 
-    # Parameters come from the environment, OS_*
-    try:
-        conn = openstack.connect()
-    except keystoneauth1.exceptions.auth_plugins.MissingRequiredOptions:
-        pass
-
     with open("/etc/conductor.yml") as fp:
         configuration = yaml.load(fp, Loader=yaml.SafeLoader)
 
@@ -33,20 +32,20 @@ def celery_init_worker(**kwargs):
             logger.warning(
                 "The conductor configuration is empty. That's probably wrong"
             )
+            configuration = {}
             return
 
         # Resolve all IDs in the conductor.yml
-        if Config.enable_ironic in ["True", "true", "Yes", "yes"]:
+        if Config.enable_ironic.lower() in ["true", "yes"]:
             if "ironic_parameters" not in configuration:
                 logger.error(
                     "ironic_parameters not found in the conductor configuration"
                 )
                 return
 
-            # TODO: use osism.tasks.openstack in the future
             if "driver_info" in configuration["ironic_parameters"]:
                 if "deploy_kernel" in configuration["ironic_parameters"]["driver_info"]:
-                    result = conn.image.find_image(
+                    result = openstack.image_get(
                         configuration["ironic_parameters"]["driver_info"][
                             "deploy_kernel"
                         ]
@@ -59,7 +58,7 @@ def celery_init_worker(**kwargs):
                     "deploy_ramdisk"
                     in configuration["ironic_parameters"]["driver_info"]
                 ):
-                    result = conn.image.find_image(
+                    result = openstack.image_get(
                         configuration["ironic_parameters"]["driver_info"][
                             "deploy_ramdisk"
                         ]
@@ -72,7 +71,7 @@ def celery_init_worker(**kwargs):
                     "cleaning_network"
                     in configuration["ironic_parameters"]["driver_info"]
                 ):
-                    result = conn.network.find_network(
+                    result = openstack.network_get(
                         configuration["ironic_parameters"]["driver_info"][
                             "cleaning_network"
                         ]
@@ -85,7 +84,7 @@ def celery_init_worker(**kwargs):
                     "provisioning_network"
                     in configuration["ironic_parameters"]["driver_info"]
                 ):
-                    result = conn.network.find_network(
+                    result = openstack.network_get(
                         configuration["ironic_parameters"]["driver_info"][
                             "provisioning_network"
                         ]
@@ -94,6 +93,49 @@ def celery_init_worker(**kwargs):
                         "provisioning_network"
                     ] = result.id
 
+    global nb_device_query_list
+
+    try:
+        supported_nb_device_filters = [
+            "site",
+            "region",
+            "site_group",
+            "location",
+            "rack",
+            "tag",
+            "state",
+        ]
+        nb_device_query_list = yaml.safe_load(
+            settings.OSISM_CONDUCTOR_NETBOX_FILTER_LIST
+        )
+        if type(nb_device_query_list) is not list:
+            raise TypeError
+        for nb_device_query in nb_device_query_list:
+            if type(nb_device_query) is not dict:
+                raise TypeError
+            for key in list(nb_device_query.keys()):
+                if key not in supported_nb_device_filters:
+                    raise ValueError
+                # NOTE: Only "location_id" and "rack_id" are supported by netbox
+                if key in ["location", "rack"]:
+                    value_name = nb_device_query.pop(key, "")
+                    if key == "location":
+                        value_id = netbox.get_location_id(value_name)
+                    elif key == "rack":
+                        value_id = netbox.get_rack_id(value_name)
+                    if value_id:
+                        nb_device_query.update({key + "_id": value_id})
+                    else:
+                        raise ValueError(f"Invalid name {value_name} for {key}")
+    except (yaml.YAMLError, TypeError):
+        logger.error(
+            f"Setting OSISM_CONDUCTOR_NETBOX_FILTER_LIST needs to be an array of mappings containing supported netbox device filters: {supported_nb_device_filters}"
+        )
+        nb_device_query_list = []
+    except ValueError as exc:
+        logger.error(f"Unknown value in OSISM_CONDUCTOR_NETBOX_FILTER_LIST: {exc}")
+        nb_device_query_list = []
+
 
 @app.on_after_configure.connect
 def setup_periodic_tasks(sender, **kwargs):
@@ -103,6 +145,322 @@ def setup_periodic_tasks(sender, **kwargs):
 @app.task(bind=True, name="osism.tasks.conductor.get_ironic_parameters")
 def get_ironic_parameters(self):
     if "ironic_parameters" in configuration:
-        return configuration["ironic_parameters"]
+        # NOTE: Do not pass by reference, everybody gets their own copy to work with
+        return copy.deepcopy(configuration["ironic_parameters"])
 
     return {}
+
+
+@app.task(bind=True, name="osism.tasks.conductor.sync_netbox_with_ironic")
+def sync_netbox_with_ironic(self, force_update=False):
+    def deep_compare(a, b, updates):
+        """
+        Find items in a that do not exist in b or are different.
+        Write required changes into updates
+        """
+        for key, value in a.items():
+            if type(value) is not dict:
+                if key not in b or b[key] != value:
+                    updates[key] = value
+            else:
+                updates[key] = {}
+                deep_compare(a[key], b[key], updates[key])
+                if not updates[key]:
+                    updates.pop(key)
+
+    driver_params = {
+        "ipmi": {
+            "address": "ipmi_address",
+            "port": "ipmi_port",
+            "password": "ipmi_password",
+        },
+        "redfish": {
+            "address": "redfish_address",
+            "password": "redfish_password",
+        },
+    }
+
+    devices = set()
+    for nb_device_query in nb_device_query_list:
+        devices |= set(netbox.get_devices(**nb_device_query))
+
+    # NOTE: Find nodes in Ironic which are no longer present in netbox and remove them
+    device_names = {dev.name for dev in devices}
+    nodes = openstack.baremetal_node_list()
+    for node in nodes:
+        logger.info(f"Looking for {node['Name']} in netbox")
+        if node["Name"] not in device_names:
+            if (
+                not node["Instance UUID"]
+                and node["Provisioning State"] in ["enroll", "manageable", "available"]
+                and node["Power State"] in ["power off", None]
+            ):
+                logger.info(
+                    f"Cleaning up baremetal node not found in netbox: {node['Name']}"
+                )
+                flavor_name = "osism-" + node["Name"]
+                flavor = openstack.compute_flavor_get(flavor_name)
+                if flavor:
+                    logger.info(f"Deleting flavor {flavor_name}")
+                    openstack.compute_flavor_delete(flavor)
+                for port in openstack.baremetal_port_list(
+                    details=False, attributes=dict(node_uuid=node["UUID"])
+                ):
+                    openstack.baremetal_port_delete(port.id)
+                openstack.baremetal_node_delete(node["UUID"])
+            else:
+                logger.error(
+                    f"Cannot remove baremetal node because it is still provisioned or running: {node}"
+                )
+
+    # NOTE: Find nodes in netbox which are not present in Ironic and add them
+    for device in devices:
+        logger.info(f"Looking for {device.name} in ironic")
+
+        node_interfaces = list(netbox.get_interfaces_by_device(device.name))
+
+        node_attributes = get_ironic_parameters()
+        if (
+            "driver" in node_attributes
+            and node_attributes["driver"] in driver_params.keys()
+        ):
+            if "driver_info" in node_attributes:
+                address_key = driver_params[node_attributes["driver"]]["address"]
+                if address_key in node_attributes["driver_info"]:
+                    if "oob_address" in device.custom_fields:
+                        node_mgmt_address = device.custom_fields["oob_address"]
+                    elif "address" in device.oob_ip:
+                        node_mgmt_address = device.oob_ip["address"]
+                    else:
+                        node_mgmt_addresses = [
+                            interface["address"]
+                            for interface in node_interfaces
+                            if interface.mgmt_only
+                            and "address" in interface
+                            and interface["address"]
+                        ]
+                        if len(node_mgmt_addresses) > 0:
+                            node_mgmt_address = node_mgmt_addresses[0]
+                        else:
+                            node_mgmt_address = None
+                    if node_mgmt_address:
+                        node_attributes["driver_info"][address_key] = (
+                            jinja2.Environment(loader=jinja2.BaseLoader())
+                            .from_string(node_attributes["driver_info"][address_key])
+                            .render(
+                                remote_board_address=str(
+                                    ipaddress.ip_interface(node_mgmt_address).ip
+                                )
+                            )
+                        )
+                    else:
+                        logger.error(f"Could not find out-of-band address for {device}")
+                        node_attributes["driver_info"].pop(address_key, None)
+                if (
+                    "port" in driver_params[node_attributes["driver"]]
+                    and "oob_port" in device.custom_fields
+                    and device.custom_fields["oob_port"]
+                ):
+                    port_key = driver_params[node_attributes["driver"]]["port"]
+                    node_attributes["driver_info"].update(
+                        {port_key: device.custom_fields["oob_port"]}
+                    )
+        node_attributes.update({"resource_class": device.name})
+        ports_attributes = [
+            dict(address=interface.mac_address)
+            for interface in node_interfaces
+            if interface.enabled and not interface.mgmt_only and interface.mac_address
+        ]
+        flavor_attributes = {
+            "ram": 1,
+            "disk": 0,
+            "vcpus": 1,
+            "is_public": False,
+            "extra_specs": {
+                "resources:CUSTOM_"
+                + device.name.upper().replace("-", "_").replace(".", "_"): "1",
+                "resources:VCPU": "0",
+                "resources:MEMORY_MB": "0",
+                "resources:DISK_GB": "0",
+            },
+        }
+
+        lock = Redlock(
+            key=f"lock_osism_tasks_conductor_sync_netbox_with_ironic-{device.name}",
+            masters={utils.redis},
+            auto_release_time=60,
+        )
+        if lock.acquire(timeout=20):
+            try:
+                logger.info(f"Processing device {device.name}")
+                node = openstack.baremetal_node_show(device.name, ignore_missing=True)
+                if not node:
+                    logger.info(f"Creating baremetal node for {device.name}")
+                    node = openstack.baremetal_node_create(device.name, node_attributes)
+                else:
+                    # NOTE: The listener service only reacts to changes in the baremetal node. Explicitly sync provision and power state in case updates were missed by the listener.
+                    if (
+                        device.custom_fields["provision_state"]
+                        != node["provision_state"]
+                    ):
+                        netbox.set_provision_state(device.name, node["provision_state"])
+                    if device.custom_fields["power_state"] != node["power_state"]:
+                        netbox.set_power_state(device.name, node["power_state"])
+                    # NOTE: Check whether the baremetal node needs to be updated
+                    node_updates = {}
+                    deep_compare(node_attributes, node, node_updates)
+                    if "driver_info" in node_updates:
+                        # NOTE: The password is not returned by ironic, so we cannot make a comparision and it would always be updated. Therefore we pop it from the dictionary
+                        password_key = driver_params[node_attributes["driver"]][
+                            "password"
+                        ]
+                        if password_key in node_updates["driver_info"]:
+                            node_updates["driver_info"].pop(password_key, None)
+                            if not node_updates["driver_info"]:
+                                node_updates.pop("driver_info", None)
+                    if node_updates or force_update:
+                        logger.info(
+                            f"Updating baremetal node for {device.name} with {node_updates}"
+                        )
+                        # NOTE: Do the actual updates with all values in node_attributes. Otherwise nested dicts like e.g. driver_info will be overwritten as a whole and contain only changed values
+                        node = openstack.baremetal_node_update(
+                            node["uuid"], node_attributes
+                        )
+
+                node_ports = openstack.baremetal_port_list(
+                    details=False, attributes=dict(node_uuid=node["uuid"])
+                )
+                # NOTE: Baremetal ports are only required for (i)pxe boot
+                if node["boot_interface"] in ["pxe", "ipxe"]:
+                    for port_attributes in ports_attributes:
+                        port_attributes.update({"node_id": node["uuid"]})
+                        port = [
+                            port
+                            for port in node_ports
+                            if port_attributes["address"].upper()
+                            == port["address"].upper()
+                        ]
+                        if not port:
+                            logger.info(
+                                f"Creating baremetal port with MAC address {port_attributes['address']} for {device.name}"
+                            )
+                            openstack.baremetal_port_create(port_attributes)
+                        else:
+                            node_ports.remove(port[0])
+                for node_port in node_ports:
+                    # NOTE: Delete remaining ports not found in netbox
+                    logger.info(
+                        f"Deleting baremetal port with MAC address {node_port['address']} for {device.name}"
+                    )
+                    openstack.baremetal_port_delete(node_port["id"])
+
+                node_validation = openstack.baremetal_node_validate(node["uuid"])
+                if node_validation["management"].result:
+                    logger.info(
+                        f"Validation of management interface successful for baremetal node for {device.name}"
+                    )
+                    if node["provision_state"] == "enroll":
+                        logger.info(
+                            f"Transitioning baremetal node to manageable state for {device.name}"
+                        )
+                        node = openstack.baremetal_node_set_provision_state(
+                            node["uuid"], "manage"
+                        )
+                        node = openstack.baremetal_node_wait_for_nodes_provision_state(
+                            node["uuid"], "manageable"
+                        )
+                        logger.info(f"Baremetal node for {device.name} is manageable")
+                    if node_validation["boot"].result:
+                        logger.info(
+                            f"Validation of boot interface successful for baremetal node for {device.name}"
+                        )
+                        if node["provision_state"] == "manageable":
+                            logger.info(
+                                f"Transitioning baremetal node to available state for {device.name}"
+                            )
+                            node = openstack.baremetal_node_set_provision_state(
+                                node["uuid"], "provide"
+                            )
+                            node = (
+                                openstack.baremetal_node_wait_for_nodes_provision_state(
+                                    node["uuid"], "available"
+                                )
+                            )
+                            logger.info(
+                                f"Baremetal node for {device.name} is available"
+                            )
+                    else:
+                        logger.info(
+                            f"Validation of boot interface failed for baremetal node for {device.name}\nReason: {node_validation['boot'].reason}"
+                        )
+                        if node["provision_state"] == "available":
+                            # NOTE: Demote node to manageable
+                            logger.info(
+                                f"Transitioning baremetal node to manageable state for {device.name}"
+                            )
+                            node = openstack.baremetal_node_set_provision_state(
+                                node["uuid"], "manage"
+                            )
+                            node = (
+                                openstack.baremetal_node_wait_for_nodes_provision_state(
+                                    node["uuid"], "manageable"
+                                )
+                            )
+                            logger.info(
+                                f"Baremetal node for {device.name} is manageable"
+                            )
+                else:
+                    logger.info(
+                        f"Validation of management interface failed for baremetal node for {device.name}\nReason: {node_validation['management'].reason}"
+                    )
+
+                flavor_name = "osism-" + device.name
+                flavor = openstack.compute_flavor_get(flavor_name)
+                if not flavor:
+                    logger.info(f"Creating flavor for {flavor_name}")
+                    flavor = openstack.compute_flavor_create(
+                        flavor_name, flavor_attributes
+                    )
+                else:
+                    flavor_updates = {}
+                    deep_compare(flavor_attributes, flavor, flavor_updates)
+                    flavor_updates_extra_specs = flavor_updates.pop("extra_specs", None)
+                    if flavor_updates:
+                        logger.info(
+                            f"Updating flavor for {device.name} with {flavor_updates}"
+                        )
+                        openstack.compute_flavor_delete(flavor)
+                        flavor = openstack.compute_flavor_create(
+                            flavor_name, flavor_attributes
+                        )
+                    elif flavor_updates_extra_specs:
+                        logger.info(
+                            f"Updating flavor extra_specs for {device.name} with {flavor_updates_extra_specs}"
+                        )
+                        openstack.compute_flavor_update_extra_specs(
+                            flavor, flavor_updates_extra_specs
+                        )
+                        flavor = openstack.compute_flavor_get(flavor_name)
+                    for extra_specs_key in flavor["extra_specs"].keys():
+                        if (
+                            extra_specs_key
+                            not in flavor_attributes["extra_specs"].keys()
+                        ):
+                            logger.info(
+                                f"Deleting flavor extra_specs property {extra_specs_key} for {device.name}"
+                            )
+                            flavor = (
+                                openstack.compute_flavor_delete_extra_specs_property(
+                                    flavor, extra_specs_key
+                                )
+                            )
+
+            except Exception as exc:
+                logger.info(
+                    f"Could not fully synchronize device {device.name} with ironic: {exc}"
+                )
+            finally:
+                lock.release()
+
+        else:
+            logger.error("Could not acquire lock for node {device.name}")