osism 0.20250331.0__py3-none-any.whl → 0.20250425.0__py3-none-any.whl

osism/tasks/netbox.py

@@ -1,126 +1,154 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from celery import Celery
-from celery.signals import worker_process_init
-import json
-import pynetbox
+from loguru import logger
+from pottery import Redlock
 
 from osism import settings, utils
-from osism.actions import manage_device, manage_interface
-from osism.tasks import Config, openstack, run_command
+from osism.tasks import Config, run_command
 
 app = Celery("netbox")
 app.config_from_object(Config)
 
 
-@worker_process_init.connect
-def celery_init_worker(**kwargs):
-    if settings.NETBOX_URL and settings.NETBOX_TOKEN:
-        utils.nb = pynetbox.api(settings.NETBOX_URL, token=settings.NETBOX_TOKEN)
-
-        if settings.IGNORE_SSL_ERRORS:
-            import requests
-
-            requests.packages.urllib3.disable_warnings()
-            session = requests.Session()
-            session.verify = False
-            utils.nb.http_session = session
-
-
 @app.on_after_configure.connect
 def setup_periodic_tasks(sender, **kwargs):
     pass
 
 
-@app.task(bind=True, name="osism.tasks.netbox.periodic_synchronize_ironic")
-def periodic_synchronize_ironic(self):
-    """Synchronize the state of Ironic with Netbox"""
-    openstack.baremetal_node_list.apply_async((), link=synchronize_device_state.s())
-
-
 @app.task(bind=True, name="osism.tasks.netbox.run")
 def run(self, action, arguments):
     pass
 
 
-@app.task(bind=True, name="osism.tasks.netbox.update_network_interface_name")
-def update_network_interface_name(self, mac_address, network_interface_name):
-    manage_interface.update_network_interface_name(mac_address, network_interface_name)
-
-
-@app.task(bind=True, name="osism.tasks.netbox.synchronize_device_state")
-def synchronize_device_state(self, data):
-    """Synchronize the state of Ironic with Netbox"""
-
-    if type(data) == str:
-        data = json.loads(data)
-
-    if not data:
-        return
-
-    for device in data:
-        manage_device.set_provision_state(device["Name"], device["Provisioning State"])
-        manage_device.set_power_state(device["Name"], device["Power State"])
-
-
-@app.task(bind=True, name="osism.tasks.netbox.states")
-def states(self, data):
-    result = manage_device.get_states(data.keys())
-    return result
-
-
-@app.task(bind=True, name="osism.tasks.netbox.set_state")
-def set_state(self, device=None, state=None, state_type=None):
-    manage_device.set_state(device, state, state_type)
+# NOTE: While `get_*` tasks only operate on the netbox configured in NETBOX_URL, `set_*` tasks additionally operate on all netbox instances listed in NETBOX_SECONDARIES
 
 
 @app.task(bind=True, name="osism.tasks.netbox.set_maintenance")
-def set_maintenance(self, device=None, state=None):
-    manage_device.set_maintenance(device, state)
-
+def set_maintenance(self, device_name, state=True):
+    """Set the maintenance state for a device in the Netbox."""
 
-@app.task(bind=True, name="osism.tasks.netbox.diff")
-@app.task(bind=True, name="osism.tasks.netbox.get_devices_not_yet_registered_in_ironic")
-def get_devices_not_yet_registered_in_ironic(
-    self, status="active", tags=["managed-by-ironic"], ironic_enabled=True
-):
-    devices = utils.nb.dcim.devices.filter(
-        tag=tags, status=status, cf_ironic_enabled=[ironic_enabled]
+    lock = Redlock(
+        key=f"lock_osism_tasks_netbox_set_maintenance_{device_name}",
+        masters={utils.redis},
+        auto_release_time=60,
     )
-
-    result = []
-
-    for device in devices:
-        if (
-            "ironic_state" in device.custom_fields
-            and device.custom_fields["ironic_state"] != "registered"
-        ):
-            result.append(device.name)
-
-    return result
-
-
-@app.task(
-    bind=True,
-    name="osism.tasks.netbox.get_devices_that_should_have_an_allocation_in_ironic",
-)
-def get_devices_that_should_have_an_allocation_in_ironic(self):
-    devices = utils.nb.dcim.devices.filter(
-        tag=["managed-by-ironic", "managed-by-osism"],
-        status="active",
-        cf_ironic_enabled=[True],
-        cf_ironic_state=["registered"],
-        cf_provision_state=["available"],
-        cf_introspection_state=["introspected"],
-        cf_device_type=["server"],
+    if lock.acquire(timeout=20):
+        try:
+            for nb in [utils.nb] + utils.secondary_nb_list:
+                logger.info(
+                    f"Set maintenance state of device {device_name} = {state} on {nb.base_url}"
+                )
+                device = nb.dcim.devices.get(name=device_name)
+                if device:
+                    device.custom_fields.update({"maintenance": state})
+                    device.save()
+                else:
+                    logger.error(
+                        f"Could not set maintenance for {device_name} on {nb.base_url}"
+                    )
+        finally:
+            lock.release()
+    else:
+        logger.error("Could not acquire lock for node {device_name}")
+
+
+@app.task(bind=True, name="osism.tasks.netbox.set_provision_state")
+def set_provision_state(self, device_name, state):
+    """Set the provision state for a device in the Netbox."""
+
+    lock = Redlock(
+        key=f"lock_osism_tasks_netbox_set_provision_state_{device_name}",
+        masters={utils.redis},
+        auto_release_time=60,
     )
-
-    result = []
-
-    for device in devices:
-        result.append(device.name)
-
-    return result
+    if lock.acquire(timeout=20):
+        try:
+
+            for nb in [utils.nb] + utils.secondary_nb_list:
+                logger.info(
+                    f"Set provision state of device {device_name} = {state} on {nb.base_url}"
+                )
+                device = nb.dcim.devices.get(name=device_name)
+                if device:
+                    device.custom_fields.update({"provision_state": state})
+                    device.save()
+                else:
+                    logger.error(
+                        f"Could not set provision state for {device_name} on {nb.base_url}"
+                    )
+        finally:
+            lock.release()
+    else:
+        logger.error("Could not acquire lock for node {device_name}")
+
+
+@app.task(bind=True, name="osism.tasks.netbox.set_power_state")
+def set_power_state(self, device_name, state):
+    """Set the provision state for a device in the Netbox."""
+
+    lock = Redlock(
+        key=f"lock_osism_tasks_netbox_set_provision_state_{device_name}",
+        masters={utils.redis},
+        auto_release_time=60,
+    )
+    if lock.acquire(timeout=20):
+        try:
+            for nb in [utils.nb] + utils.secondary_nb_list:
+                logger.info(
+                    f"Set power state of device {device_name} = {state} on {nb.base_url}"
+                )
+                device = nb.dcim.devices.get(name=device_name)
+                if device:
+                    device.custom_fields.update({"power_state": state})
+                    device.save()
+                else:
+                    logger.error(
+                        f"Could not set power state for {device_name} on {nb.base_url}"
+                    )
+        finally:
+            lock.release()
+    else:
+        logger.error("Could not acquire lock for node {device_name}")
+
+
+@app.task(bind=True, name="osism.tasks.netbox.get_location_id")
+def get_location_id(self, location_name):
+    try:
+        location = utils.nb.dcim.locations.get(name=location_name)
+    except ValueError:
+        return None
+    if location:
+        return location.id
+    else:
+        return None
+
+
+@app.task(bind=True, name="osism.tasks.netbox.get_rack_id")
+def get_rack_id(self, rack_name):
+    try:
+        rack = utils.nb.dcim.racks.get(name=rack_name)
+    except ValueError:
+        return None
+    if rack:
+        return rack.id
+    else:
+        return None
+
+
+@app.task(bind=True, name="osism.tasks.netbox.get_devices")
+def get_devices(self, **query):
+    return utils.nb.dcim.devices.filter(**query)
+
+
+@app.task(bind=True, name="osism.tasks.netbox.get_device_by_name")
+def get_device_by_name(self, name):
+    return utils.nb.dcim.devices.get(name=name)
+
+
+@app.task(bind=True, name="osism.tasks.netbox.get_interfaces_by_device")
+def get_interfaces_by_device(self, device_name):
+    return utils.nb.dcim.interfaces.filter(device=device_name)
 
 
 @app.task(bind=True, name="osism.tasks.netbox.manage")
@@ -139,7 +167,7 @@ def manage(self, *arguments, publish=True, locking=False, auto_release_time=3600
         *arguments,
         publish=publish,
         locking=locking,
-        auto_release_time=auto_release_time
+        auto_release_time=auto_release_time,
     )
 
 

osism/tasks/openstack.py

@@ -1,16 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
-import copy
-import ipaddress
-
 from celery import Celery
-import jinja2
-from openstack.exceptions import ConflictException, ResourceNotFound, ResourceFailure
-from pottery import Redlock
 import tempfile
 
 from osism import utils
-from osism.tasks import Config, conductor, netbox, run_command
+from osism.tasks import Config, run_command
 
 app = Celery("openstack")
 app.config_from_object(Config)
@@ -25,20 +19,46 @@ def setup_periodic_tasks(sender, **kwargs):
 def image_get(self, image_name):
     conn = utils.get_openstack_connection()
     result = conn.image.find_image(image_name)
-    return result.id
+    return result
 
 
 @app.task(bind=True, name="osism.tasks.openstack.network_get")
 def network_get(self, network_name):
     conn = utils.get_openstack_connection()
     result = conn.network.find_network(network_name)
-    return result.id
+    return result
+
+
+@app.task(bind=True, name="osism.tasks.openstack.baremetal_node_create")
+def baremetal_node_create(self, node_name, attributes=None):
+    if attributes is None:
+        attributes = {}
+    attributes.update({"name": node_name})
+    conn = utils.get_openstack_connection()
+    result = conn.baremetal.create_node(**attributes)
+    return result
+
+
+@app.task(bind=True, name="osism.tasks.openstack.baremetal_node_delete")
+def baremetal_node_delete(self, node_or_id):
+    conn = utils.get_openstack_connection()
+    result = conn.baremetal.delete_node(node_or_id)
+    return result
+
+
+@app.task(bind=True, name="osism.tasks.openstack.baremetal_node_update")
+def baremetal_node_update(self, node_id_or_name, attributes=None):
+    if attributes is None:
+        attributes = {}
+    conn = utils.get_openstack_connection()
+    result = conn.baremetal.update_node(node_id_or_name, **attributes)
+    return result
 
 
 @app.task(bind=True, name="osism.tasks.openstack.baremetal_node_show")
-def baremetal_node_show(self, node_id_or_name):
+def baremetal_node_show(self, node_id_or_name, ignore_missing=False):
     conn = utils.get_openstack_connection()
-    result = conn.baremetal.find_node(node_id_or_name)
+    result = conn.baremetal.find_node(node_id_or_name, ignore_missing)
     return result
 
 
@@ -64,182 +84,108 @@ def baremetal_node_list(self):
     return result
 
 
-@app.task(
-    bind=True, name="osism.tasks.openstack.baremetal_introspection_interface_list"
-)
-def baremetal_introspection_interface_list(self, node_id_or_name):
-    pass
-
-
-@app.task(bind=True, name="osism.tasks.openstack.baremetal_introspection_status")
-def baremetal_introspection_status(self, node_id_or_name):
-    result = None
+@app.task(bind=True, name="osism.tasks.openstack.baremetal_node_validate")
+def baremetal_node_validate(self, node_id_or_name):
+    conn = utils.get_openstack_connection()
+    result = conn.baremetal.validate_node(node_id_or_name, required=())
     return result
 
 
-@app.task(bind=True, name="osism.tasks.openstack.baremetal_get_network_interface_name")
-def baremetal_get_network_interface_name(self, node_name, mac_address):
+@app.task(
+    bind=True,
+    name="osism.tasks.openstack.baremetal_node_wait_for_nodes_provision_state",
+)
+def baremetal_node_wait_for_nodes_provision_state(self, node_id_or_name, state):
     conn = utils.get_openstack_connection()
+    result = conn.baremetal.wait_for_nodes_provision_state([node_id_or_name], state)
+    if len(result) > 0:
+        return result[0]
+    else:
+        return None
 
-    introspection = conn.baremetal_introspection.get_introspection(node_name)
-
-    # Wait up to 5 minutes for the completion of a running introspection
-    conn.baremetal_introspection.wait_for_introspection(introspection, timeout=30)
-
-    introspection_data = conn.baremetal_introspection.get_introspection_data(
-        introspection
-    )
-    interfaces = introspection_data["inventory"]["interfaces"]
-
-    result = None
-    for interface in interfaces:
-        if interface["mac_address"].lower() == mac_address.lower():
-            result = interface["name"]
 
+@app.task(bind=True, name="osism.tasks.openstack.baremetal_node_set_provision_state")
+def baremetal_node_set_provision_state(self, node, state):
+    conn = utils.get_openstack_connection()
+    result = conn.baremetal.set_node_provision_state(node, state)
     return result
 
 
-@app.task(bind=True, name="osism.tasks.openstack.baremetal_set_node_provision_state")
-def baremetal_set_node_provision_state(self, node, state):
+@app.task(bind=True, name="osism.tasks.openstack.baremetal_port_list")
+def baremetal_port_list(self, details=False, attributes=None):
+    if attributes is None:
+        attributes = {}
     conn = utils.get_openstack_connection()
-    conn.baremetal.set_node_provision_state(node, state)
+    result = conn.baremetal.ports(details=details, **attributes)
+    return list(result)
 
 
-@app.task(bind=True, name="osism.tasks.openstack.baremetal_create_allocations")
-def baremetal_create_allocations(self, nodes):
+@app.task(bind=True, name="osism.tasks.openstack.baremetal_port_create")
+def baremetal_port_create(self, attributes=None):
+    if attributes is None:
+        attributes = {}
     conn = utils.get_openstack_connection()
-
-    for node in nodes:
-        try:
-            allocation_a = conn.baremetal.get_allocation(allocation=node)
-        except ResourceNotFound:
-            allocation_a = None
-
-        if not allocation_a:
-            # Get Ironic parameters from the conductor
-            task = conductor.get_ironic_parameters.delay()
-            task.wait(timeout=None, interval=0.5)
-            ironic_parameters = task.get()
-
-            allocation_a = conn.baremetal.create_allocation(
-                name=node,
-                candidate_nodes=[node],
-                resource_class=ironic_parameters["resource_class"],
-            )
-            conn.baremetal.wait_for_allocation(allocation=node, timeout=30)
+    result = conn.baremetal.create_port(**attributes)
+    return result
 
 
-@app.task(bind=True, name="osism.tasks.openstack.baremetal_create_nodes")
-def baremetal_create_nodes(self, nodes, ironic_parameters):
+@app.task(bind=True, name="osism.tasks.openstack.baremetal_port_delete")
+def baremetal_port_delete(self, port_or_id):
     conn = utils.get_openstack_connection()
+    result = conn.baremetal.delete_port(port_or_id)
+    return result
 
-    for node in nodes:
-        # TODO: Filter on mgmt_only
-        address_a = utils.nb.ipam.ip_addresses.get(device=node, interface="Ethernet0")
-
-        node_parameters = copy.deepcopy(ironic_parameters)
-
-        if node_parameters["driver"] == "redfish":
-            remote_board_address = str(ipaddress.ip_interface(address_a["address"]).ip)
-            t = jinja2.Environment(loader=jinja2.BaseLoader()).from_string(
-                node_parameters["driver_info"]["redfish_address"]
-            )
-            node_parameters["driver_info"]["redfish_address"] = t.render(
-                remote_board_address=remote_board_address
-            )
-
-        elif node_parameters["driver"] == "ipmi":
-            remote_board_address = str(ipaddress.ip_interface(address_a["address"]).ip)
-            t = jinja2.Environment(loader=jinja2.BaseLoader()).from_string(
-                node_parameters["driver_info"]["ipmi_address"]
-            )
-            node_parameters["driver_info"]["ipmi_address"] = t.render(
-                remote_board_address=remote_board_address
-            )
-
-        try:
-            device_a = utils.nb.dcim.devices.get(name=node)
-            tags = [str(tag) for tag in device_a.tags]
-
-            # NOTE: Internally used nodes are identified by their unique name via the resource class.
-            #       The actual resource class is explicitly overwritten.
-            if "Managed by Ironic" in tags and "Managed by OSISM" in tags:
-                node_parameters["resource_class"] = f"osism-{node}"
-                baremetal_create_internal_flavor(node)
-
-            conn.baremetal.create_node(
-                name=node, provision_state="manageable", **node_parameters
-            )
-            conn.baremetal.wait_for_nodes_provision_state([node], "manageable")
-
-            if "Managed by Ironic" in tags and "Managed by OSISM" not in tags:
-                conn.baremetal.set_node_traits(node, ["CUSTOM_GENERAL_USE"])
-            elif "Managed by Ironic" in tags and "Managed by OSISM" in tags:
-                conn.baremetal.set_node_traits(node, ["CUSTOM_OSISM_USE"])
 
-            conn.baremetal.set_node_provision_state(node, "inspect")
+@app.task(bind=True, name="osism.tasks.openstack.compute_flavor_get")
+def compute_flavor_get(self, name_or_id):
+    conn = utils.get_openstack_connection()
+    result = conn.compute.find_flavor(
+        name_or_id, ignore_missing=True, get_extra_specs=True
+    )
+    return result
 
-            # TODO: Check if the system has been registered correctly
-            device_a.custom_fields = {
-                "ironic_state": "registered",
-            }
-            device_a.save()
-
-        except ResourceFailure:
-            # TODO: Do something useful here
-            pass
-        except ConflictException:
-            # The node already exists and has a wronge state in the Netbox
-            device_a = utils.nb.dcim.devices.get(name=node)
-            device_a.custom_fields = {
-                "ironic_state": "registered",
-            }
-            device_a.save()
 
+@app.task(bind=True, name="osism.tasks.openstack.compute_flavor_create")
+def compute_flavor_create(self, name, attributes=None):
+    if attributes is None:
+        attributes = {}
+    attributes.update({"name": name})
+    extra_specs = attributes.pop("extra_specs", None)
+    conn = utils.get_openstack_connection()
+    flavor = conn.compute.create_flavor(**attributes)
+    if extra_specs:
+        flavor = conn.compute.create_flavor_extra_specs(flavor, extra_specs)
+    return flavor
 
-@app.task(bind=True, name="osism.tasks.openstack.baremetal_check_allocations")
-def baremetal_check_allocations(self):
-    lock = Redlock(
-        key="lock_osism_tasks_openstack_baremetal_check_allocations",
-        masters={utils.redis},
-        auto_release_time=60,
-    )
 
-    if lock.acquire(timeout=20):
-        netbox.get_devices_that_should_have_an_allocation_in_ironic.apply_async(
-            (), link=baremetal_create_allocations.s()
-        )
-        lock.release()
+@app.task(bind=True, name="osism.tasks.openstack.compute_flavor_delete")
+def compute_flavor_delete(self, flavor):
+    conn = utils.get_openstack_connection()
+    conn.compute.delete_flavor(flavor, ignore_missing=True)
 
 
-@app.task(bind=True, name="osism.tasks.openstack.baremetal_create_internal_flavor")
-def baremetal_create_internal_flavor(self, node):
+@app.task(bind=True, name="osism.tasks.openstack.compute_flavor_update_extra_specs")
+def compute_flavor_update_extra_specs(self, flavor, extra_specs={}):
     conn = utils.get_openstack_connection()
+    for key, value in extra_specs.items():
+        conn.compute.update_flavor_extra_specs_property(flavor, key, value)
 
-    flavor_a = conn.compute.create_flavor(
-        name=f"osism-{node}", ram=1, vcpus=1, disk=1, is_public=False
-    )
-    specs = {
-        f"resources:CUSTOM_RESOURCE_CLASS_OSISM_{node.upper()}": 1,
-        "resources:VCPU": 0,
-        "resources:MEMORY_MB": 0,
-        "resources:DISK_GB": 0,
-        "trait:CUSTOM_OSISM_USE": "required",
-    }
-    conn.compute.create_flavor_extra_specs(flavor_a, specs)
-
-
-@app.task(bind=True, name="osism.tasks.openstack.baremetal_delete_internal_flavor")
-def baremetal_delete_internal_flavor(self, node):
-    conn = utils.get_openstack_connection()
 
-    flavor = conn.compute.get_flavor(f"osism-{node}")
-    conn.compute.delete_flavor(flavor)
+@app.task(bind=True, name="osism.tasks.openstack.compute_flavor_delete_extra_specs")
+def compute_flavor_delete_extra_specs_property(self, flavor, prop):
+    conn = utils.get_openstack_connection()
+    conn.compute.delete_flavor_extra_specs_property(flavor, prop)
 
 
 @app.task(bind=True, name="osism.tasks.openstack.image_manager")
 def image_manager(
-    self, *arguments, configs=None, publish=True, locking=False, auto_release_time=3600
+    self,
+    *arguments,
+    configs=None,
+    publish=True,
+    locking=False,
+    auto_release_time=3600,
+    ignore_env=False
 ):
     command = "/usr/local/bin/openstack-image-manager"
     if configs:
@@ -269,6 +215,7 @@ def image_manager(
                 publish=publish,
                 locking=locking,
                 auto_release_time=auto_release_time,
+                ignore_env=ignore_env,
             )
         return rc
     else:
@@ -280,6 +227,7 @@ def image_manager(
             publish=publish,
             locking=locking,
             auto_release_time=auto_release_time,
+            ignore_env=ignore_env,
         )