osism 0.20250219.0__py3-none-any.whl → 0.20250314.0__py3-none-any.whl

osism/services/listener.py

@@ -2,14 +2,18 @@
 
 import os
 import time
+from collections.abc import Callable
+from typing import Any
 
 from kombu import Connection, Exchange, Queue
 from kombu.mixins import ConsumerMixin
 from loguru import logger
 import json
+import requests
 
 from osism import utils
 from osism.tasks import netbox, openstack
+from osism import settings
 
 EXCHANGE_NAME = "ironic"
 ROUTING_KEY = "ironic_versioned_notifications.info"
@@ -17,168 +21,254 @@ QUEUE_NAME = "osism-listener-ironic"
 BROKER_URI = os.getenv("BROKER_URI")
 
 
-class NotificationsDump(ConsumerMixin):
-    def __init__(self, connection):
-        self.connection = connection
-        return
-
-    def get_consumers(self, consumer, channel):
-        exchange = Exchange(EXCHANGE_NAME, type="topic", durable=False)
-        queue = Queue(
-            QUEUE_NAME,
-            exchange,
-            routing_key=ROUTING_KEY,
-            durable=False,
-            auto_delete=True,
-            no_ack=True,
-        )
-        return [consumer(queue, callbacks=[self.on_message])]
+class BaremetalEvents:
+    # References:
+    #
+    # * https://docs.openstack.org/ironic/latest/admin/notifications.html#available-notifications
+    # * https://docs.openstack.org/ironic/latest/user/states.html
+    # * https://docs.openstack.org/ironic/latest/_images/states.svg
+
+    def __init__(self) -> None:
+        self._handler: dict[
+            str, dict[str, dict[str, dict[str, Callable[[dict[Any, Any]], None]]]]
+        ] = {
+            "baremetal": {
+                "node": {
+                    "power_set": {"end": self.node_power_set_end},
+                    "power_state_corrected": {
+                        "success": self.node_power_state_corrected_success
+                    },
+                    "maintenance_set": {"end": self.node_maintenance_set_end},
+                    "provision_set": {
+                        "end": self.node_provision_set_end,
+                        "success": self.node_provision_set_success,
+                    },
+                    "delete": {"end": self.node_delete_end},
+                    "create": {"end": self.node_create_end},
+                },
+                "port": {
+                    "create": {"end": self.port_create_end},
+                    "update": {"end": self.port_update_end},
+                },
+            }
+        }
+
+    def get_object_data(self, payload: dict[Any, Any]) -> Any:
+        return payload["ironic_object.data"]
+
+    def get_handler(self, event_type: str) -> Callable[[dict[Any, Any]], None]:
+        event_type_keys = event_type.split(".")
+        try:
+            handler = self._handler[event_type_keys[0]][event_type_keys[1]][
+                event_type_keys[2]
+            ][event_type_keys[3]]
+        except KeyError:
 
-    def on_message(self, body, message):
-        data = json.loads(body["oslo.message"])
-        # logger.info(data)
+            def default_handler(payload: dict[Any, Any]) -> None:
+                object_data = self.get_object_data(payload)
+                name = object_data["name"]
+                logger.info(event_type + f" ## {name}")
 
-        event_type = data["event_type"]
-        payload = data["payload"]
-        object_data = payload["ironic_object.data"]
+            handler = default_handler
+        return handler
 
+    def node_power_set_end(self, payload: dict[Any, Any]) -> None:
+        object_data = self.get_object_data(payload)
         name = object_data["name"]
+        logger.info(
+            f"baremetal.node.power_set.end ## {name} ## {object_data['power_state']}"
+        )
+        netbox.set_state.delay(name, object_data["power_state"], "power")
 
-        # References:
-        #
-        # * https://docs.openstack.org/ironic/latest/admin/notifications.html#available-notifications
-        # * https://docs.openstack.org/ironic/latest/user/states.html
-        # * https://docs.openstack.org/ironic/latest/_images/states.svg
+    def node_power_state_corrected_success(self, payload: dict[Any, Any]) -> None:
+        object_data = self.get_object_data(payload)
+        name = object_data["name"]
+        logger.info(
+            f"baremetal.node.power_state_corrected.success ## {name} ## {object_data['power_state']}"
+        )
+        netbox.set_state.delay(name, object_data["power_state"], "power")
 
-        if event_type == "baremetal.node.power_set.end":
-            logger.info(
-                f"baremetal.node.power_set.end ## {name} ## {object_data['power_state']}"
-            )
-            netbox.set_state.delay(name, object_data["power_state"], "power")
+    def node_maintenance_set_end(self, payload: dict[Any, Any]) -> None:
+        object_data = self.get_object_data(payload)
+        name = object_data["name"]
+        logger.info(
+            f"baremetal.node.maintenance_set.end ## {name} ## {object_data['maintenance']}"
+        )
+        netbox.set_maintenance.delay(name, object_data["maintenance"])
 
-        elif event_type == "baremetal.node.power_state_corrected.success":
-            logger.info(
-                f"baremetal.node.power_state_corrected.success ## {name} ## {object_data['power_state']}"
-            )
-            netbox.set_state.delay(name, object_data["power_state"], "power")
+    def node_provision_set_success(self, payload: dict[Any, Any]) -> None:
+        # A provision status was successfully set, update it in the netbox
+        object_data = self.get_object_data(payload)
+        name = object_data["name"]
+        logger.info(
+            f"baremetal.node.provision_set.success ## {name} ## {object_data['provision_state']}"
+        )
+        netbox.set_state.delay(name, object_data["provision_state"], "provision")
 
-        elif event_type == "baremetal.node.maintenance_set.end":
-            logger.info(
-                f"baremetal.node.maintenance_set.end ## {name} ## {object_data['maintenance']}"
-            )
-            netbox.set_maintenance.delay(name, object_data["maintenance"])
+    def node_provision_set_end(self, payload: dict[Any, Any]) -> None:
+        object_data = self.get_object_data(payload)
+        name = object_data["name"]
+        logger.info(
+            f"baremetal.node.provision_set.end ## {name} ## {object_data['provision_state']}"
+        )
+        netbox.set_state.delay(name, object_data["provision_state"], "provision")
 
-        elif event_type == "baremetal.node.provision_set.start":
-            logger.info(
-                f"baremetal.node.provision_set.start ## {name} ## {object_data['provision_state']}"
-            )
+        if (
+            object_data["previous_provision_state"] == "inspect wait"
+            and object_data["event"] == "done"
+        ):
+            netbox.set_state.delay(name, "introspected", "introspection")
+            openstack.baremetal_set_node_provision_state.delay(name, "provide")
 
-            if object_data["event"] == "inspect":
-                # system should be in state a
-                netbox.connect.delay(name, "a")
+    def port_create_end(self, payload: dict[Any, Any]) -> None:
+        object_data = self.get_object_data(payload)
+        name = object_data["name"]
+        logger.info(f"baremetal.port.create.end ## {object_data['uuid']}")
 
-            if object_data["provision_state"] == "cleaning":
-                # system should be in state b
-                netbox.connect.delay(name, "b")
+        mac_address = object_data["address"]
+        interface_a = utils.nb.dcim.interfaces.get(mac_address=mac_address)
+        device_a = interface_a.device
 
-            if object_data["provision_state"] == "available":
-                # system should be in state c
-                netbox.connect.delay(name, "c")
+        task = openstack.baremetal_get_network_interface_name.delay(
+            device_a.name, mac_address
+        )
+        task.wait(timeout=None, interval=0.5)
+        network_interface_name = task.get()
 
-            if object_data["target_provision_state"] == "active":
-                pass
+        netbox.update_network_interface_name.delay(
+            object_data["address"], network_interface_name
+        )
 
-        # A provision status was successfully set, update it in the netbox
-        elif event_type == "baremetal.node.provision_set.success":
-            logger.info(
-                f"baremetal.node.provision_set.success ## {name} ## {object_data['provision_state']}"
-            )
-            netbox.set_state.delay(name, object_data["provision_state"], "provision")
+    def port_update_end(self, payload: dict[Any, Any]) -> None:
+        object_data = self.get_object_data(payload)
+        name = object_data["name"]
+        logger.info(f"baremetal.port.update.end ## {object_data['uuid']}")
 
-            if object_data["provision_state"] == "manageable":
-                # system should be in state c
-                netbox.connect.delay(name, "c")
+        mac_address = object_data["address"]
+        interface_a = utils.nb.dcim.interfaces.get(mac_address=mac_address)
+        device_a = interface_a.device
 
-        elif event_type == "baremetal.node.provision_set.end":
-            logger.info(
-                f"baremetal.node.provision_set.end ## {name} ## {object_data['provision_state']}"
-            )
-            netbox.set_state.delay(name, object_data["provision_state"], "provision")
-
-            if (
-                object_data["previous_provision_state"] == "inspect wait"
-                and object_data["event"] == "done"
-            ):
-                netbox.set_state.delay(name, "introspected", "introspection")
-                openstack.baremetal_set_node_provision_state.delay(name, "provide")
-
-            elif object_data["previous_provision_state"] == "wait call-back":
-                pass
-
-            elif (
-                object_data["previous_provision_state"] == "cleaning"
-                and object_data["provision_state"] == "available"
-            ):  # noqa
-                # system should be in state c
-                netbox.connect.delay(name, "c")
-
-        elif event_type == "baremetal.port.create.end":
-            logger.info(f"baremetal.port.create.end ## {object_data['uuid']}")
-
-            mac_address = object_data["address"]
-            interface_a = utils.nb.dcim.interfaces.get(mac_address=mac_address)
-            device_a = interface_a.device
-
-            task = openstack.baremetal_get_network_interface_name.delay(
-                device_a.name, mac_address
-            )
-            task.wait(timeout=None, interval=0.5)
-            network_interface_name = task.get()
+        task = openstack.baremetal_get_network_interface_name.delay(
+            device_a.name, mac_address
+        )
+        task.wait(timeout=None, interval=0.5)
+        network_interface_name = task.get()
 
-            netbox.update_network_interface_name.delay(
-                object_data["address"], network_interface_name
-            )
+        netbox.update_network_interface_name.delay(
+            object_data["address"], network_interface_name
+        )
 
-        elif event_type == "baremetal.port.update.end":
-            logger.info(f"baremetal.port.update.end ## {object_data['uuid']}")
+    def node_delete_end(self, payload: dict[Any, Any]) -> None:
+        object_data = self.get_object_data(payload)
+        name = object_data["name"]
+        logger.info(f"baremetal.node.delete.end ## {name}")
 
-            mac_address = object_data["address"]
-            interface_a = utils.nb.dcim.interfaces.get(mac_address=mac_address)
-            device_a = interface_a.device
+        netbox.set_state.delay(name, "unregistered", "ironic")
+        netbox.set_state.delay(name, None, "provision")
+        netbox.set_state.delay(name, None, "power")
+        netbox.set_state.delay(name, None, "introspection")
+        netbox.set_state.delay(name, None, "deployment")
 
-            task = openstack.baremetal_get_network_interface_name.delay(
-                device_a.name, mac_address
-            )
-            task.wait(timeout=None, interval=0.5)
-            network_interface_name = task.get()
+        # remove internal flavor
+        openstack.baremetal_delete_internal_flavor.delay(name)
 
-            netbox.update_network_interface_name.delay(
-                object_data["address"], network_interface_name
-            )
+    def node_create_end(self, payload: dict[Any, Any]) -> None:
+        object_data = self.get_object_data(payload)
+        name = object_data["name"]
+        logger.info(f"baremetal.node.create.end ## {name}")
+        netbox.set_state.delay(name, "registered", "ironic")
 
-        elif event_type == "baremetal.node.delete.end":
-            logger.info(f"baremetal.node.delete.end ## {name}")
 
-            netbox.set_state.delay(name, "unregistered", "ironic")
-            netbox.set_state.delay(name, None, "provision")
-            netbox.set_state.delay(name, None, "power")
-            netbox.set_state.delay(name, None, "introspection")
-            netbox.set_state.delay(name, None, "deployment")
+class NotificationsDump(ConsumerMixin):
+    def __init__(self, connection):
+        self.connection = connection
+        self.baremetal_events = BaremetalEvents()
+        self.osism_api_session: None | requests.Session = None
+        self.osism_baremetal_api_url: None | str = None
+        if settings.OSISM_API_URL:
+            logger.info("Setting up OSISM API")
+            self.osism_api_session = requests.Session()
+            self.osism_baremetal_api_url = (
+                settings.OSISM_API_URL.rstrip("/") + "/notifications/baremetal"
+            )
+        return
 
-            # system should be in state a
-            netbox.connect.delay(name, "a")
+    def get_consumers(self, consumer, channel):
+        exchange = Exchange(EXCHANGE_NAME, type="topic", durable=False)
+        queue = Queue(
+            QUEUE_NAME,
+            exchange,
+            routing_key=ROUTING_KEY,
+            durable=False,
+            auto_delete=True,
+            no_ack=True,
+        )
+        return [consumer(queue, callbacks=[self.on_message])]
 
-            # remove internal flavor
-            openstack.baremetal_delete_internal_flavor.delay(name)
+    def on_message(self, body, message):
+        data = json.loads(body["oslo.message"])
+        # logger.info(data)
 
-        elif event_type == "baremetal.node.create.end":
-            logger.info(f"baremetal.node.create.end ## {name}")
-            netbox.set_state.delay(name, "registered", "ironic")
+        if self.osism_api_session:
+            tries = 1
+            max_tries = 3
+            while tries <= max_tries:
+                logger.info(
+                    f"Trying to deliver notification to {self.osism_baremetal_api_url} (Try: {tries}/{max_tries})\n"
+                )
+                try:
+                    response = self.osism_api_session.post(
+                        self.osism_baremetal_api_url,
+                        timeout=5,
+                        json=dict(
+                            priority=data["priority"],
+                            event_type=data["event_type"],
+                            timestamp=data["timestamp"],
+                            publisher_id=data["publisher_id"],
+                            message_id=data["message_id"],
+                            payload=data["payload"],
+                        ),
+                    )
+                    if response.status_code == 204:
+                        logger.info(
+                            f"Successfully delivered notification to {self.osism_baremetal_api_url} (Try: {tries}/{max_tries})"
+                        )
+                        break
+                    else:
+                        response.raise_for_status()
+                except requests.ConnectionError:
+                    logger.error(f"Error connecting to {self.osism_baremetal_api_url}")
+                except requests.Timeout:
+                    logger.error(
+                        f"Timeout reached while connecting to {self.osism_baremetal_api_url}"
+                    )
+                except requests.HTTPError as e:
+                    logger.error(
+                        f"Received HTTP status code {e.response.status_code} while connecting to {self.osism_baremetal_api_url}"
+                    )
+                    if e.response.status_code <= 500:
+                        logger.error(
+                            f"Received HTTP status code {e.response.status_code} indicates a client side error, giving up early"
+                        )
+                        break
+
+                logger.error(
+                    f"Failed to deliver notification to {self.osism_baremetal_api_url} ({tries}/{max_tries})"
+                )
+                tries += 1
+                if tries > max_tries:
+                    logger.error(
+                        f"Giving up delivering notification to {self.osism_baremetal_api_url} with data:\n"
+                        + json.dumps(data)
+                    )
+                else:
+                    time.sleep(pow(3, tries - 1))
 
         else:
-            logger.info(f"{event_type} ## {name}")
+            handler = self.baremetal_events.get_handler(data["event_type"])
+            handler(data["payload"])
 
-        logger.info(object_data)
+        logger.info(self.baremetal_events.get_object_data(data["payload"]))
 
 
 def main():

osism/settings.py

@@ -27,11 +27,10 @@ NETBOX_URL = os.getenv("NETBOX_API")
 NETBOX_TOKEN = os.getenv("NETBOX_TOKEN", read_secret("NETBOX_TOKEN"))
 IGNORE_SSL_ERRORS = os.getenv("IGNORE_SSL_ERRORS", "True") == "True"
 
-BASE_PATH = os.getenv("BASE_PATH", "/devicetype-library/device-types/")
-VENDORS = os.getenv("VENDORS", "").split()
-
 # 43200 seconds = 12 hours
 GATHER_FACTS_SCHEDULE = float(os.getenv("GATHER_FACTS_SCHEDULE", "43200.0"))
 INVENTORY_RECONCILER_SCHEDULE = float(
     os.getenv("INVENTORY_RECONCILER_SCHEDULE", "600.0")
 )
+
+OSISM_API_URL = os.getenv("OSISM_API_URL", None)

osism/tasks/__init__.py

@@ -160,20 +160,6 @@ def run_ansible_in_environment(
             env=env,
         )
 
-    # execute local netbox playbooks
-    elif worker == "osism-ansible" and environment == "netbox-local":
-        if locking:
-            lock.acquire()
-
-        command = f"/run-{environment}.sh {role} {joined_arguments}"
-        logger.info(f"RUN {command}")
-        p = subprocess.Popen(
-            command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            shell=True,
-        )
-
     # execute all other roles
     else:
         if locking:
@@ -207,6 +193,50 @@ def run_ansible_in_environment(
     return result
 
 
+def run_command(
+    request_id,
+    command,
+    env,
+    *arguments,
+    publish=True,
+    locking=False,
+    auto_release_time=3600,
+):
+    result = ""
+    command_env = os.environ.copy()
+    command_env.update(env)
+
+    if locking:
+        lock = Redlock(
+            key=f"lock-{command}",
+            masters={redis},
+            auto_release_time=auto_release_time,
+        )
+
+    p = subprocess.Popen(
+        [command] + list(arguments),
+        env=command_env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    while p.poll() is None:
+        line = p.stdout.readline().decode("utf-8")
+        if publish:
+            redis.xadd(request_id, {"type": "stdout", "content": line})
+        result += line
+
+    rc = p.wait(timeout=60)
+
+    if publish:
+        redis.xadd(request_id, {"type": "rc", "content": rc})
+        redis.xadd(request_id, {"type": "action", "content": "quit"})
+
+    if locking:
+        lock.release()
+
+    return result
+
+
 def handle_task(t, wait=True, format="log", timeout=3600):
     global redis
 

osism/tasks/netbox.py

@@ -1,8 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
 
-import os
-import subprocess
-
 from celery import Celery
 from celery.signals import worker_process_init
 import json
@@ -10,15 +7,8 @@ import pynetbox
 from redis import Redis
 
 from osism import settings
-from osism.actions import (
-    check_configuration,
-    deploy_configuration,
-    diff_configuration,
-    generate_configuration,
-    manage_device,
-    manage_interface,
-)
-from osism.tasks import Config, openstack
+from osism.actions import manage_device, manage_interface
+from osism.tasks import Config, openstack, run_command
 
 app = Celery("netbox")
 app.config_from_object(Config)
@@ -73,35 +63,6 @@ def update_network_interface_name(self, mac_address, network_interface_name):
     manage_interface.update_network_interface_name(mac_address, network_interface_name)
 
 
-@app.task(bind=True, name="osism.tasks.netbox.import_device_types")
-def import_device_types(self, vendors, library=False):
-    global redis
-
-    if library:
-        env = {**os.environ, "BASE_PATH": "/devicetype-library/device-types/"}
-    else:
-        env = {**os.environ, "BASE_PATH": "/netbox/device-types/"}
-
-    if vendors:
-        p = subprocess.Popen(
-            f"python3 /import/main.py --vendors {vendors}",
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            env=env,
-        )
-    else:
-        p = subprocess.Popen(
-            "python3 /import/main.py",
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            env=env,
-        )
-
-    p.communicate()
-
-
 @app.task(bind=True, name="osism.tasks.netbox.synchronize_device_state")
 def synchronize_device_state(self, data):
     """Synchronize the state of Ironic with Netbox"""
@@ -123,23 +84,6 @@ def states(self, data):
     return result
 
 
-@app.task(bind=True, name="osism.tasks.netbox.transitions")
-def transitions(self, data):
-    result = manage_device.get_transitions(data.keys())
-    return result
-
-
-@app.task(bind=True, name="osism.tasks.netbox.data")
-def data(self, collection, device, state):
-    result = manage_device.load_data_from_filesystem(collection, device, state)
-    return result
-
-
-@app.task(bind=True, name="osism.tasks.netbox.connect")
-def connect(self, device=None, state=None, data={}, enforce=False):
-    manage_device.run(device, state, data, enforce)
-
-
 @app.task(bind=True, name="osism.tasks.netbox.set_state")
 def set_state(self, device=None, state=None, state_type=None):
     manage_device.set_state(device, state, state_type)
@@ -150,47 +94,7 @@ def set_maintenance(self, device=None, state=None):
     manage_device.set_maintenance(device, state)
 
 
-@app.task(bind=True, name="osism.tasks.netbox.disable")
-def disable(self, name):
-    global nb
-
-    for interface in nb.dcim.interfaces.filter(device=name):
-        if str(interface.type) in ["Virtual"]:
-            continue
-
-        if "Port-Channel" in interface.name:
-            continue
-
-        if not interface.connected_endpoint and interface.enabled:
-            interface.enabled = False
-            interface.save()
-
-        # FIXME: only enable devices that are not disabled by configuration
-        if interface.connected_endpoint and not interface.enabled:
-            interface.enabled = True
-            interface.save()
-
-
-@app.task(bind=True, name="osism.tasks.netbox.generate")
-def generate(self, name, template=None):
-    generate_configuration.for_device(name, template)
-
-
-@app.task(bind=True, name="osism.tasks.netbox.deploy")
-def deploy(self, name):
-    deploy_configuration.for_device(name)
-
-
-@app.task(bind=True, name="osism.tasks.netbox.check")
-def check(self, name):
-    check_configuration.for_device(name)
-
-
 @app.task(bind=True, name="osism.tasks.netbox.diff")
-def diff(self, name):
-    diff_configuration.for_device(name)
-
-
 @app.task(bind=True, name="osism.tasks.netbox.get_devices_not_yet_registered_in_ironic")
 def get_devices_not_yet_registered_in_ironic(
     self, status="active", tags=["managed-by-ironic"], ironic_enabled=True
@@ -238,6 +142,26 @@ def get_devices_that_should_have_an_allocation_in_ironic(self):
     return result
 
 
+@app.task(bind=True, name="osism.tasks.netbox.manage")
+def manage(self, *arguments, publish=True, locking=False, auto_release_time=3600):
+    netbox_manager_env = {
+        "NETBOX_MANAGER_URL": str(settings.NETBOX_URL),
+        "NETBOX_MANAGER_TOKEN": str(settings.NETBOX_TOKEN),
+        "NETBOX_MANAGER_IGNORE_SSL_ERRORS": str(settings.IGNORE_SSL_ERRORS),
+        "NETBOX_MANAGER_VERBOSE": "true",
+    }
+
+    return run_command(
+        self.request.id,
+        "/usr/local/bin/netbox-manager",
+        netbox_manager_env,
+        *arguments,
+        publish=publish,
+        locking=locking,
+        auto_release_time=auto_release_time
+    )
+
+
 @app.task(bind=True, name="osism.tasks.netbox.ping")
 def ping(self):
     global nb

osism-0.20250314.0.dist-info/AUTHORS

@@ -0,0 +1 @@
+janhorstmann <horstmann@osism.tech>