osism 0.20250605.0__py3-none-any.whl → 0.20250621.0__py3-none-any.whl

osism/commands/baremetal.py

@@ -2,10 +2,14 @@
 
 from cliff.command import Command
 
+import tempfile
+import os
 from loguru import logger
 import openstack
 from tabulate import tabulate
 import json
+import yaml
+from openstack.baremetal import configdrive as configdrive_builder
 
 from osism.commands import get_cloud_connection
 
@@ -127,20 +131,35 @@ class BaremetalDeploy(Command):
             if not node:
                 continue
 
-            if node.provision_state in ["available", "deploy failed"]:
+            if (
+                node.provision_state in ["available", "deploy failed"]
+                and not node["maintenance"]
+            ):
                 provision_state = "active"
             elif (
                 node.provision_state == "error"
                 or node.provision_state == "active"
+                and not node["maintenance"]
                 and rebuild
             ):
                 provision_state = "rebuild"
             else:
                 logger.warning(
-                    f"Node {node.name} ({node.id}) not in supported provision state"
+                    f"Node {node.name} ({node.id}) not in supported state! Provision state: {node.provision_state}, maintenance mode: {node['maintenance']}"
                 )
                 continue
 
+            # NOTE: Ironic removes "instance_info" on undeploy. It was saved to "extra" during sync and needs to be refreshed here.
+            if (
+                "instance_info" in node
+                and not node["instance_info"]
+                and "instance_info" in node["extra"]
+                and node["extra"]["instance_info"]
+            ):
+                node = conn.baremetal.update_node(
+                    node, instance_info=json.loads(node.extra["instance_info"])
+                )
+
             try:
                 conn.baremetal.validate_node(
                     node.id, required=("boot", "deploy", "power")
@@ -148,23 +167,60 @@ class BaremetalDeploy(Command):
             except openstack.exceptions.ValidationException:
                 logger.warning(f"Node {node.name} ({node.id}) could not be validated")
                 continue
+            # NOTE: Prepare osism config drive
             try:
-                config_drive = {"meta_data": {}}
+                playbook = []
+                play = {
+                    "name": "Run bootstrap - part 2",
+                    "hosts": "localhost",
+                    "connection": "local",
+                    "gather_facts": True,
+                    "vars": {},
+                    "roles": [
+                        "osism.commons.hostname",
+                        "osism.commons.hosts",
+                    ],
+                }
+                play["vars"].update(
+                    {"hostname_name": node.name, "hosts_type": "template"}
+                )
                 if (
                     "netplan_parameters" in node.extra
                     and node.extra["netplan_parameters"]
                 ):
-                    config_drive["meta_data"].update(
+                    play["vars"].update(
                         {
-                            "netplan_parameters": json.loads(
-                                node.extra["netplan_parameters"]
-                            )
+                            "network_allow_service_restart": True,
                         }
                     )
+                    play["vars"].update(json.loads(node.extra["netplan_parameters"]))
+                    play["roles"].append("osism.commons.network")
                 if "frr_parameters" in node.extra and node.extra["frr_parameters"]:
-                    config_drive["meta_data"].update(
-                        {"frr_parameters": json.loads(node.extra["frr_parameters"])}
+                    play["vars"].update(
+                        {
+                            "frr_dummy_interface": "loopback0",
+                        }
                     )
+                    play["vars"].update(json.loads(node.extra["frr_parameters"]))
+                    play["roles"].append("osism.services.frr")
+                playbook.append(play)
+                with tempfile.TemporaryDirectory() as tmp_dir:
+                    with open(os.path.join(tmp_dir, "playbook.yml"), "w") as file:
+                        yaml.dump(
+                            playbook,
+                            file,
+                            default_flow_style=False,
+                            explicit_start=True,
+                            indent=2,
+                            sort_keys=False,
+                        )
+                    config_drive = configdrive_builder.pack(tmp_dir)
+            except Exception as exc:
+                logger.warning(
+                    f"Failed to build config drive for {node.name} ({node.id}): {exc}"
+                )
+                continue
+            try:
                 conn.baremetal.set_node_provision_state(
                     node.id, provision_state, config_drive=config_drive
                 )
@@ -231,12 +287,17 @@ class BaremetalUndeploy(Command):
 
             if node.provision_state in ["active", "deploy failed", "error"]:
                 try:
-                    conn.baremetal.set_node_provision_state(node.id, "undeploy")
+                    node = conn.baremetal.set_node_provision_state(node.id, "undeploy")
                 except Exception as exc:
                     logger.warning(
                         f"Node {node.name} ({node.id}) could not be moved to available state: {exc}"
                     )
                     continue
+                # NOTE: Ironic removes "instance_info" on undeploy. It was saved to "extra" during sync and needs to be refreshed here.
+                if "instance_info" in node["extra"]:
+                    node = conn.baremetal.update_node(
+                        node, instance_info=json.loads(node.extra["instance_info"])
+                    )
             else:
                 logger.warning(
                     f"Node {node.name} ({node.id}) not in supported provision state"

osism/commands/manage.py

@@ -10,7 +10,7 @@ from loguru import logger
 import requests
 
 from osism.data import TEMPLATE_IMAGE_CLUSTERAPI, TEMPLATE_IMAGE_OCTAVIA
-from osism.tasks import openstack, handle_task
+from osism.tasks import openstack, ansible, handle_task
 
 SUPPORTED_CLUSTERAPI_K8S_IMAGES = ["1.31", "1.32", "1.33"]
 
@@ -360,3 +360,27 @@ class Flavors(Command):
             )
 
         return handle_task(task, wait, format="script", timeout=3600)
+
+
+class Dnsmasq(Command):
+    def get_parser(self, prog_name):
+        parser = super(Dnsmasq, self).get_parser(prog_name)
+        parser.add_argument(
+            "--no-wait",
+            default=False,
+            help="Do not wait until dnsmasq has been applied",
+            action="store_true",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        wait = not parsed_args.no_wait
+
+        task_signature = ansible.run.si("infrastructure", "dnsmasq", [])
+        task = task_signature.apply_async()
+        if wait:
+            logger.info(
+                f"It takes a moment until task {task.task_id} (dnsmasq) has been started and output is visible here."
+            )
+
+        return handle_task(task, wait, format="log", timeout=300)

osism/commands/netbox.py

@@ -8,6 +8,7 @@ from loguru import logger
 import yaml
 
 from osism.tasks import conductor, netbox, handle_task
+from osism import utils
 
 
 class Ironic(Command):
@@ -18,6 +19,12 @@ class Ironic(Command):
             help="Do not wait until the sync has been completed",
             action="store_true",
         )
+        parser.add_argument(
+            "--task-timeout",
+            default=os.environ.get("OSISM_TASK_TIMEOUT", 300),
+            type=int,
+            help="Timeout for a scheduled task that has not been executed yet",
+        )
         parser.add_argument(
             "--force-update",
             help="Force update of baremetal nodes (Used to update non-comparable items like passwords)",
@@ -27,13 +34,23 @@ class Ironic(Command):
 
     def take_action(self, parsed_args):
         wait = not parsed_args.no_wait
+        task_timeout = parsed_args.task_timeout
 
         task = conductor.sync_ironic.delay(force_update=parsed_args.force_update)
         if wait:
             logger.info(
-                f"Task {task.task_id} (sync ironic) is running. Wait. No more output."
+                f"Task {task.task_id} (sync ironic) is running in background. Output comming soon."
+            )
+            try:
+                return utils.fetch_task_output(task.id, timeout=task_timeout)
+            except TimeoutError:
+                logger.error(
+                    f"Timeout while waiting for further output of task {task.task_id} (sync ironic)"
+                )
+        else:
+            logger.info(
+                f"Task {task.task_id} (sync ironic) is running in background. No more output."
             )
-            task.wait(timeout=None, interval=0.5)
 
 
 class Sync(Command):
@@ -63,13 +80,13 @@ class Manage(Command):
         parser.add_argument(
             "--no-wait",
             default=False,
-            help="Do not wait until the management of the netbox has been completed",
+            help="Do not wait until the management of the NetBox has been completed",
             action="store_true",
         )
         parser.add_argument(
             "--no-netbox-wait",
             default=False,
-            help="Do not wait for the netbox API to be ready",
+            help="Do not wait for the NetBox API to be ready",
             action="store_true",
         )
         parser.add_argument(
@@ -195,7 +212,7 @@ class Console(Command):
             url = os.environ.get("NETBOX_API", None)
 
             if not token or not url:
-                logger.error("Netbox integration not configured.")
+                logger.error("NetBox integration not configured.")
                 return
 
             subprocess.call(

osism/commands/reconciler.py

@@ -2,13 +2,12 @@
 
 import os
 import subprocess
-import time
 
 from cliff.command import Command
 from loguru import logger
 
 from osism.tasks import reconciler
-from osism.utils import redis
+from osism import utils
 
 
 class Run(Command):
@@ -50,33 +49,12 @@ class Sync(Command):
             logger.info(
                 f"Task {t.task_id} (sync inventory) is running in background. Output coming soon."
             )
-            rc = 0
-            stoptime = time.time() + task_timeout
-            last_id = 0
-            while time.time() < stoptime:
-                data = redis.xread(
-                    {str(t.task_id): last_id}, count=1, block=(300 * 1000)
+            try:
+                return utils.fetch_task_output(t.id, timeout=task_timeout)
+            except TimeoutError:
+                logger.error(
+                    f"Timeout while waiting for further output of task {t.task_id} (sync inventory)"
                 )
-                if data:
-                    stoptime = time.time() + task_timeout
-                    messages = data[0]
-                    for message_id, message in messages[1]:
-                        last_id = message_id.decode()
-                        message_type = message[b"type"].decode()
-                        message_content = message[b"content"].decode()
-
-                        logger.debug(
-                            f"Processing message {last_id} of type {message_type}"
-                        )
-                        redis.xdel(str(t.task_id), last_id)
-
-                        if message_type == "stdout":
-                            print(message_content, end="")
-                        elif message_type == "rc":
-                            rc = int(message_content)
-                        elif message_type == "action" and message_content == "quit":
-                            redis.close()
-                            return rc
         else:
             logger.info(
                 f"Task {t.task_id} (sync inventory) is running in background. No more output."

osism/commands/sync.py

@@ -1,8 +1,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from cliff.command import Command
+from loguru import logger
 
-from osism.tasks import ansible, handle_task
+from osism.tasks import ansible, conductor, handle_task
 
 
 class Facts(Command):
@@ -17,3 +18,49 @@ class Facts(Command):
         )
         rc = handle_task(t)
         return rc
+
+
+class Sonic(Command):
+    def get_parser(self, prog_name):
+        parser = super(Sonic, self).get_parser(prog_name)
+        parser.add_argument(
+            "device",
+            nargs="?",
+            help="Optional device name to sync configuration for a specific device",
+        )
+        parser.add_argument(
+            "--no-wait",
+            default=False,
+            help="Do not wait until the sync has been completed",
+            action="store_true",
+        )
+        parser.add_argument(
+            "--diff",
+            default=True,
+            help="Show configuration diff when changes are detected (default: True)",
+            action="store_true",
+        )
+        parser.add_argument(
+            "--no-diff",
+            dest="diff",
+            help="Do not show configuration diff",
+            action="store_false",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        wait = not parsed_args.no_wait
+        device_name = parsed_args.device
+        show_diff = parsed_args.diff
+
+        task = conductor.sync_sonic.delay(device_name, show_diff)
+
+        if device_name:
+            logger.info(
+                f"Task {task.task_id} (sync sonic for device {device_name}) started"
+            )
+        else:
+            logger.info(f"Task {task.task_id} (sync sonic) started")
+
+        rc = handle_task(task, wait=wait)
+        return rc

osism/commands/validate.py

@@ -1,14 +1,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
-import time
 
 from cliff.command import Command
 from loguru import logger
 
 from osism.core.enums import VALIDATE_PLAYBOOKS
 from osism.tasks import ansible, ceph, kolla
-from osism.utils import redis
+from osism import utils
 
 
 class Run(Command):
@@ -53,35 +52,13 @@ class Run(Command):
         return parser
 
     def _handle_task(self, t, wait, format, timeout, playbook):
-        rc = 0
         if wait:
-            stoptime = time.time() + timeout
-            last_id = 0
-            while time.time() < stoptime:
-                data = redis.xread(
-                    {str(t.task_id): last_id}, count=1, block=(timeout * 1000)
+            try:
+                return utils.fetch_task_output(t.id, timeout=timeout)
+            except TimeoutError:
+                logger.error(
+                    f"Timeout while waiting for further output of task {t.task_id} (sync inventory)"
                 )
-                if data:
-                    stoptime = time.time() + timeout
-                    messages = data[0]
-                    for message_id, message in messages[1]:
-                        last_id = message_id.decode()
-                        message_type = message[b"type"].decode()
-                        message_content = message[b"content"].decode()
-
-                        logger.debug(
-                            f"Processing message {last_id} of type {message_type}"
-                        )
-                        redis.xdel(str(t.task_id), message_id)
-
-                        if message_type == "stdout":
-                            print(message_content, end="")
-                        elif message_type == "rc":
-                            rc = int(message_content)
-                        elif message_type == "action" and message_content == "quit":
-                            redis.close()
-                            return rc
-
         else:
             if format == "log":
                 logger.info(
@@ -90,7 +67,7 @@ class Run(Command):
             elif format == "script":
                 print(f"{t.task_id}")
 
-            return rc
+            return 0
 
     def take_action(self, parsed_args):
         arguments = parsed_args.arguments

osism/commands/wait.py

@@ -119,38 +119,15 @@ class Run(Command):
 
                     if live:
                         utils.redis.ping()
-
-                        last_id = 0
-                        while_True = True
-                        while while_True:
-                            data = utils.redis.xread(
-                                {str(task_id): last_id}, count=1, block=1000
+                        try:
+                            rc = utils.fetch_task_output(task_id)
+                        except TimeoutError:
+                            logger.error(
+                                f"Timeout while waiting for further output of task {task_id}"
                             )
-                            if data:
-                                messages = data[0]
-                                for message_id, message in messages[1]:
-                                    last_id = message_id.decode()
-                                    message_type = message[b"type"].decode()
-                                    message_content = message[b"content"].decode()
-
-                                    logger.debug(
-                                        f"Processing message {last_id} of type {message_type}"
-                                    )
-                                    utils.redis.xdel(str(task_id), last_id)
-
-                                    if message_type == "stdout":
-                                        print(message_content, end="")
-                                    elif message_type == "rc":
-                                        rc = int(message_content)
-                                    elif (
-                                        message_type == "action"
-                                        and message_content == "quit"
-                                    ):
-                                        utils.redis.close()
-                                        if len(task_ids) == 1:
-                                            return rc
-                                        else:
-                                            while_True = False
+
+                        if len(task_ids) == 1:
+                            return rc
                     else:
                         tmp_task_ids.insert(0, task_id)
 

osism/services/listener.py

@@ -93,7 +93,7 @@ class BaremetalEvents:
         netbox.set_maintenance.delay(name, state=object_data["maintenance"])
 
     def node_provision_set_success(self, payload: dict[Any, Any]) -> None:
-        # A provision status was successfully set, update it in the netbox
+        # A provision status was successfully set, update it in the NetBox
         object_data = self.get_object_data(payload)
         name = object_data["name"]
         logger.info(

osism/settings.py

@@ -35,11 +35,22 @@ INVENTORY_RECONCILER_SCHEDULE = float(
 
 OSISM_API_URL = os.getenv("OSISM_API_URL", None)
 
-NETBOX_FILTER_CONDUCTOR = os.getenv(
-    "NETBOX_FILTER_CONDUCTOR",
+NETBOX_FILTER_CONDUCTOR_IRONIC = os.getenv(
+    "NETBOX_FILTER_CONDUCTOR_IRONIC",
     "[{'state': 'active', 'tag': ['managed-by-ironic']}]",
 )
 
+NETBOX_FILTER_CONDUCTOR_SONIC = os.getenv(
+    "NETBOX_FILTER_CONDUCTOR_SONIC",
+    "[{'state': 'active', 'tag': ['managed-by-metalbox']}]",
+)
+
+# SONiC export configuration
+SONIC_EXPORT_DIR = os.getenv("SONIC_EXPORT_DIR", "/etc/sonic/export")
+SONIC_EXPORT_PREFIX = os.getenv("SONIC_EXPORT_PREFIX", "osism_")
+SONIC_EXPORT_SUFFIX = os.getenv("SONIC_EXPORT_SUFFIX", "_config_db.json")
+SONIC_EXPORT_IDENTIFIER = os.getenv("SONIC_EXPORT_IDENTIFIER", "serial-number")
+
 NETBOX_SECONDARIES = (
     os.getenv("NETBOX_SECONDARIES", read_secret("NETBOX_SECONDARIES")) or "[]"
 )

osism/tasks/__init__.py

@@ -4,7 +4,6 @@ import logging
 import os
 import re
 import subprocess
-import time
 
 from loguru import logger
 from pottery import Redlock
@@ -163,14 +162,13 @@ def run_ansible_in_environment(
     while p.poll() is None:
         line = p.stdout.readline().decode("utf-8")
         if publish:
-            utils.redis.xadd(request_id, {"type": "stdout", "content": line})
+            utils.push_task_output(request_id, line)
         result += line
 
     rc = p.wait(timeout=60)
 
     if publish:
-        utils.redis.xadd(request_id, {"type": "rc", "content": rc})
-        utils.redis.xadd(request_id, {"type": "action", "content": "quit"})
+        utils.finish_task_output(request_id, rc=rc)
 
     if locking:
         lock.release()
@@ -212,14 +210,13 @@ def run_command(
     while p.poll() is None:
         line = p.stdout.readline().decode("utf-8")
         if publish:
-            utils.redis.xadd(request_id, {"type": "stdout", "content": line})
+            utils.push_task_output(request_id, line)
         result += line
 
     rc = p.wait(timeout=60)
 
     if publish:
-        utils.redis.xadd(request_id, {"type": "rc", "content": rc})
-        utils.redis.xadd(request_id, {"type": "action", "content": "quit"})
+        utils.finish_task_output(request_id, rc=rc)
 
     if locking:
         lock.release()
@@ -228,39 +225,10 @@ def run_command(
 
 
 def handle_task(t, wait=True, format="log", timeout=3600):
-    rc = 0
     if wait:
-        stoptime = time.time() + timeout
-        last_id = 0
-        while time.time() < stoptime:
-            data = utils.redis.xread(
-                {str(t.task_id): last_id}, count=1, block=(timeout * 1000)
-            )
-            if data:
-                stoptime = time.time() + timeout
-                messages = data[0]
-                for message_id, message in messages[1]:
-                    last_id = message_id.decode()
-                    message_type = message[b"type"].decode()
-                    message_content = message[b"content"].decode()
-
-                    logger.debug(f"Processing message {last_id} of type {message_type}")
-                    utils.redis.xdel(str(t.task_id), last_id)
-
-                    if message_type == "stdout":
-                        print(message_content, end="", flush=True)
-                        if "PLAY RECAP" in message_content:
-                            logger.info(
-                                "Play has been completed. There may now be a delay until "
-                                "all logs have been written."
-                            )
-                            logger.info("Please wait and do not abort execution.")
-                    elif message_type == "rc":
-                        rc = int(message_content)
-                    elif message_type == "action" and message_content == "quit":
-                        utils.redis.close()
-                        return rc
-        else:
+        try:
+            return utils.fetch_task_output(t.id, timeout=timeout)
+        except TimeoutError:
             logger.info(
                 f"There has been no output from the task {t.task_id} for {timeout} second(s)."
             )
@@ -284,4 +252,4 @@ def handle_task(t, wait=True, format="log", timeout=3600):
         elif format == "script":
             print(f"{t.task_id}")
 
-        return rc
+        return 0

osism/tasks/conductor/__init__.py

@@ -8,6 +8,7 @@ from loguru import logger
 from osism.tasks import Config
 from osism.tasks.conductor.config import get_configuration
 from osism.tasks.conductor.ironic import sync_ironic as _sync_ironic
+from osism.tasks.conductor.sonic import sync_sonic as _sync_sonic
 
 
 # App configuration
@@ -43,7 +44,12 @@ def sync_netbox(self, force_update=False):
 
 @app.task(bind=True, name="osism.tasks.conductor.sync_ironic")
 def sync_ironic(self, force_update=False):
-    _sync_ironic(get_ironic_parameters, force_update)
+    _sync_ironic(self.request.id, get_ironic_parameters, force_update)
+
+
+@app.task(bind=True, name="osism.tasks.conductor.sync_sonic")
+def sync_sonic(self, device_name=None, show_diff=True):
+    return _sync_sonic(device_name, self.request.id, show_diff)
 
 
 __all__ = [
@@ -51,4 +57,5 @@ __all__ = [
     "get_ironic_parameters",
     "sync_netbox",
     "sync_ironic",
+    "sync_sonic",
 ]