osism 0.20250312.0__py3-none-any.whl → 0.20250326.0__py3-none-any.whl

osism/commands/netbox.py

@@ -1,40 +1,17 @@
 # SPDX-License-Identifier: Apache-2.0
 
-import argparse
-from glob import glob
-from os.path import basename
-
 from cliff.command import Command
 from loguru import logger
 from redis import Redis
-from tabulate import tabulate
 
 from osism import settings
-from osism.tasks import conductor, netbox, reconciler, ansible, openstack, handle_task
+from osism.tasks import conductor, netbox, reconciler, openstack, handle_task
 
 
 redis = Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_DB)
 redis.ping()
 
 
-class Run(Command):
-    def get_parser(self, prog_name):
-        parser = super(Run, self).get_parser(prog_name)
-        parser.add_argument(
-            "arguments", nargs=argparse.REMAINDER, help="Other arguments for Ansible"
-        )
-        parser.add_argument(
-            "--no-wait",
-            default=False,
-            help="Do not wait until the role has been applied",
-            action="store_true",
-        )
-        return parser
-
-    def take_action(self, parsed_args):
-        pass
-
-
 class Ironic(Command):
     def get_parser(self, prog_name):
         parser = super(Ironic, self).get_parser(prog_name)
@@ -77,368 +54,93 @@ class Sync(Command):
             task.wait(timeout=None, interval=0.5)
 
 
-class Init(Command):
-    def get_parser(self, prog_name):
-        parser = super(Init, self).get_parser(prog_name)
-        parser.add_argument(
-            "arguments", nargs=argparse.REMAINDER, help="Other arguments for Ansible"
-        )
-        parser.add_argument(
-            "--no-wait",
-            default=False,
-            help="Do not wait until the role has been applied",
-            action="store_true",
-        )
-        parser.add_argument(
-            "--format",
-            default="log",
-            help="Output type",
-            const="log",
-            nargs="?",
-            choices=["script", "log"],
-        ),
-        return parser
-
-    def take_action(self, parsed_args):
-        arguments = parsed_args.arguments
-        format = parsed_args.format
-        wait = not parsed_args.no_wait
-
-        task = ansible.run.delay("netbox-local", "init", arguments)
-        rc = 0
-
-        if wait:
-            logger.info(f"Task {task.task_id} was prepared for execution.")
-            logger.info(
-                f"It takes a moment until task {task.task_id} has been started and output is visible here."
-            )
-            rc = handle_task(task, wait, format, 300)
-        else:
-            logger.info(
-                f"Task {task.task_id} is running in background. No more output. Check ARA for logs."
-            )
-
-        return rc
-
-
-class Import(Command):
-    def get_parser(self, prog_name):
-        parser = super(Import, self).get_parser(prog_name)
-        parser.add_argument(
-            "--vendors",
-            help="Vendors from which all available device types are to be imported",
-            required=False,
-        )
-        parser.add_argument(
-            "--library",
-            default=False,
-            help="Do import device types from the device type library",
-            action="store_true",
-        )
-        parser.add_argument(
-            "--no-wait",
-            default=False,
-            help="Do not wait until the role has been applied",
-            action="store_true",
-        )
-        return parser
-
-    def take_action(self, parsed_args):
-        vendors = parsed_args.vendors
-        wait = not parsed_args.no_wait
-
-        task = netbox.import_device_types.delay(vendors, parsed_args.library)
-
-        if wait:
-            logger.info(f"Task {task.task_id} is running. Wait. No more output.")
-            task.wait(timeout=None, interval=0.5)
-
-
 class Manage(Command):
     def get_parser(self, prog_name):
         parser = super(Manage, self).get_parser(prog_name)
-        parser.add_argument(
-            "--type",
-            type=str,
-            help="Type of the resource to manage",
-            required=False,
-            default="rack",
-        )
-        parser.add_argument(
-            "name", nargs="?", type=str, help="Name of the resource to manage"
-        )
-        parser.add_argument(
-            "arguments", nargs=argparse.REMAINDER, help="Other arguments for Ansible"
-        )
         parser.add_argument(
             "--no-wait",
             default=False,
-            help="Do not wait until the changes have been made",
+            help="Do not wait until the management of the netbox has been completed",
             action="store_true",
         )
         parser.add_argument(
-            "--format",
-            default="log",
-            help="Output type",
-            const="log",
-            nargs="?",
-            choices=["script", "log"],
-        ),
-        return parser
-
-    def take_action(self, parsed_args):
-        arguments = parsed_args.arguments
-        format = parsed_args.format
-        name = parsed_args.name
-        type_of_resource = parsed_args.type
-        wait = not parsed_args.no_wait
-
-        rc = 0
-        if not name:
-            logger.info("No name of an object to be managed was given")
-            table = []
-            for playbook_type in ["rack"]:
-                playbooks = glob(
-                    f"/opt/configuration/netbox/playbooks/{playbook_type}-*.yml"
-                )
-                for playbook in playbooks:
-                    name = basename(playbook)[len(playbook_type) + 1 : -4]  # noqa E203
-                    table.append([playbook_type, name])
-
-            print(tabulate(table, headers=["Type", "Name"], tablefmt="psql"))
-        else:
-            task = ansible.run.delay(
-                "netbox-local", f"{type_of_resource}-{name}", arguments
-            )
-
-            if wait:
-                logger.info(f"Task {task.task_id} was prepared for execution.")
-                logger.info(
-                    f"It takes a moment until task {task.task_id} has been started and output is visible here."
-                )
-                rc = handle_task(task, wait, format, 300)
-            else:
-                logger.info(
-                    f"Task {task.task_id} is running in background. No more output. Check ARA for logs."
-                )
-
-        return rc
-
-
-class Connect(Command):
-    def get_parser(self, prog_name):
-        parser = super(Connect, self).get_parser(prog_name)
-        parser.add_argument(
-            "name",
-            nargs="?",
-            type=str,
-            help="Name of the resource (a collection or a device) to connect",
-        )
-        parser.add_argument(
-            "--collection",
-            type=str,
-            help="Name of the collection to connect",
-            required=False,
-        ),
-        parser.add_argument(
-            "--device", type=str, help="Name of the device to connect", required=False
-        )
-        parser.add_argument(
-            "--enforce",
+            "--no-netbox-wait",
             default=False,
-            help="Ignore the current transition of a device",
+            help="Do not wait for the netbox API to be ready",
             action="store_true",
         )
         parser.add_argument(
-            "--state", type=str, help="State to use", default=None, required=False
-        )
-        parser.add_argument(
-            "--type",
+            "--parallel",
             type=str,
-            default="collection",
-            help="Type of the resource to connection (when not using --collection or --device)",
-            required=False,
+            default=None,
+            help="Process up to n files in parallel",
         )
-        return parser
-
-    def take_action(self, parsed_args):
-        name = parsed_args.name
-        collection = parsed_args.device
-        device = parsed_args.device
-        state = parsed_args.state
-        type_of_resource = parsed_args.type
-        enforce = parsed_args.enforce
-
-        task = None
-
-        if name:
-            if type_of_resource == "collection":
-                task = netbox.data.delay(name, "", state)
-            elif type_of_resource == "device":
-                task = netbox.data.delay("", name, state)
-        else:
-            task = netbox.data.delay(collection, device, state)
-            name = f"{collection}-{device}"
-
-        task.wait(timeout=None, interval=0.5)
-        data = task.get()
-
-        for device in data:
-            t = netbox.connect.delay(device, state, data, enforce)
-            logger.info(
-                f"Task {t.task_id} for device {device} is running in background"
-            )
-        logger.info(
-            "Tasks are running in background. No more output. Check Flower for logs."
-        )
-
-
-class Disable(Command):
-    def get_parser(self, prog_name):
-        parser = super(Disable, self).get_parser(prog_name)
         parser.add_argument(
-            "name",
-            nargs=1,
+            "--limit",
             type=str,
-            help="Name of the device to check for unused interfaces",
+            default=None,
+            help="Limit files by prefix",
         )
         parser.add_argument(
-            "--no-wait",
+            "--skipdtl",
             default=False,
-            help="Do not wait until the changes have been made",
+            help="Skip devicetype library",
             action="store_true",
         )
-        return parser
-
-    def take_action(self, parsed_args):
-        name = parsed_args.name[0]
-        wait = not parsed_args.no_wait
-
-        task = netbox.disable.delay(name)
-
-        if wait:
-            logger.info(f"Task {task.task_id} is running. Wait. No more output.")
-            task.wait(timeout=None, interval=0.5)
-
-
-class Generate(Command):
-    def get_parser(self, prog_name):
-        parser = super(Generate, self).get_parser(prog_name)
-        parser.add_argument(
-            "name",
-            nargs=1,
-            type=str,
-            help="Name of the device for which the configuration is to be generated.",
-        )
         parser.add_argument(
-            "--no-wait",
+            "--skipmtl",
             default=False,
-            help="Do not wait until the changes have been made",
+            help="Skip moduletype library",
             action="store_true",
         )
         parser.add_argument(
-            "--template", type=str, help="Name of the template to use", required=False
-        )
-        return parser
-
-    def take_action(self, parsed_args):
-        name = parsed_args.name[0]
-        template = parsed_args.template
-        wait = not parsed_args.no_wait
-
-        task = netbox.generate.delay(name, template)
-
-        if wait:
-            logger.info(f"Task {task.task_id} is running. Wait. No more output.")
-            task.wait(timeout=None, interval=0.5)
-
-
-class Deploy(Command):
-    def get_parser(self, prog_name):
-        parser = super(Deploy, self).get_parser(prog_name)
-        parser.add_argument(
-            "name",
-            nargs=1,
-            type=str,
-            help="Name of the device for which the configuration is to be deployed",
-        )
-        parser.add_argument(
-            "arguments", nargs=argparse.REMAINDER, help="Other arguments for Ansible"
-        )
-        parser.add_argument(
-            "--no-wait",
+            "--skipres",
             default=False,
-            help="Do not wait until the changes have been made",
+            help="Skip resources",
             action="store_true",
         )
         return parser
 
     def take_action(self, parsed_args):
-        name = parsed_args.name[0]
-        # arguments = parsed_args.arguments
         wait = not parsed_args.no_wait
+        arguments = []
 
-        task = netbox.deploy.delay(name)
+        if parsed_args.no_netbox_wait:
+            arguments.append("--no-wait")
+        else:
+            arguments.append("--wait")
 
-        if wait:
-            logger.info(f"Task {task.task_id} is running. Wait. No more output.")
-            task.wait(timeout=None, interval=0.5)
+        if parsed_args.parallel:
+            arguments.append("--parallel")
+            arguments.append(parsed_args.parallel)
 
+        if parsed_args.limit:
+            arguments.append("--limit")
+            arguments.append(parsed_args.limit)
 
-class Check(Command):
-    def get_parser(self, prog_name):
-        parser = super(Check, self).get_parser(prog_name)
-        parser.add_argument(
-            "name",
-            nargs=1,
-            type=str,
-            help="Name of the device for which the configuration is to be checked",
-        )
-        parser.add_argument(
-            "--no-wait",
-            default=False,
-            help="Do not wait until the changes have been made",
-            action="store_true",
-        )
-        return parser
+        if parsed_args.skipdtl:
+            arguments.append("--skipdtl")
+        else:
+            arguments.append("--no-skipdtl")
 
-    def take_action(self, parsed_args):
-        name = parsed_args.name[0]
-        wait = not parsed_args.no_wait
+        if parsed_args.skipmtl:
+            arguments.append("--skipmtl")
+        else:
+            arguments.append("--no-skipmtl")
 
-        task = netbox.check.delay(name)
+        if parsed_args.skipres:
+            arguments.append("--skipres")
+        else:
+            arguments.append("--no-skipres")
 
+        task_signature = netbox.manage.si(*arguments)
+        task = task_signature.apply_async()
         if wait:
-            logger.info(f"Task {task.task_id} is running. Wait. No more output.")
-            task.wait(timeout=None, interval=0.5)
-
-
-class Diff(Command):
-    def get_parser(self, prog_name):
-        parser = super(Diff, self).get_parser(prog_name)
-        parser.add_argument(
-            "name",
-            nargs=1,
-            type=str,
-            help="Name of the device for which the configuration is to be diffed",
-        )
-        parser.add_argument(
-            "--no-wait",
-            default=False,
-            help="Do not wait until the changes have been made",
-            action="store_true",
-        )
-        return parser
-
-    def take_action(self, parsed_args):
-        name = parsed_args.name[0]
-        wait = not parsed_args.no_wait
-
-        task = netbox.diff.delay(name)
+            logger.info(
+                f"It takes a moment until task {task.task_id} (netbox-manager) has been started and output is visible here."
+            )
 
-        if wait:
-            logger.info(f"Task {task.task_id} is running. Wait. No more output.")
-            task.wait(timeout=None, interval=0.5)
+        return handle_task(task, wait, format="script", timeout=3600)
 
 
 class Ping(Command):

osism/services/listener.py

@@ -40,7 +40,6 @@ class BaremetalEvents:
                     },
                     "maintenance_set": {"end": self.node_maintenance_set_end},
                     "provision_set": {
-                        "start": self.node_provision_set_start,
                         "end": self.node_provision_set_end,
                         "success": self.node_provision_set_success,
                     },
@@ -97,28 +96,6 @@ class BaremetalEvents:
         )
         netbox.set_maintenance.delay(name, object_data["maintenance"])
 
-    def node_provision_set_start(self, payload: dict[Any, Any]) -> None:
-        object_data = self.get_object_data(payload)
-        name = object_data["name"]
-        logger.info(
-            f"baremetal.node.provision_set.start ## {name} ## {object_data['provision_state']}"
-        )
-
-        if object_data["event"] == "inspect":
-            # system should be in state a
-            netbox.connect.delay(name, "a")
-
-        if object_data["provision_state"] == "cleaning":
-            # system should be in state b
-            netbox.connect.delay(name, "b")
-
-        if object_data["provision_state"] == "available":
-            # system should be in state c
-            netbox.connect.delay(name, "c")
-
-        if object_data["target_provision_state"] == "active":
-            pass
-
     def node_provision_set_success(self, payload: dict[Any, Any]) -> None:
         # A provision status was successfully set, update it in the netbox
         object_data = self.get_object_data(payload)
@@ -128,10 +105,6 @@ class BaremetalEvents:
         )
         netbox.set_state.delay(name, object_data["provision_state"], "provision")
 
-        if object_data["provision_state"] == "manageable":
-            # system should be in state c
-            netbox.connect.delay(name, "c")
-
     def node_provision_set_end(self, payload: dict[Any, Any]) -> None:
         object_data = self.get_object_data(payload)
         name = object_data["name"]
@@ -147,16 +120,6 @@ class BaremetalEvents:
             netbox.set_state.delay(name, "introspected", "introspection")
             openstack.baremetal_set_node_provision_state.delay(name, "provide")
 
-        elif object_data["previous_provision_state"] == "wait call-back":
-            pass
-
-        elif (
-            object_data["previous_provision_state"] == "cleaning"
-            and object_data["provision_state"] == "available"
-        ):  # noqa
-            # system should be in state c
-            netbox.connect.delay(name, "c")
-
     def port_create_end(self, payload: dict[Any, Any]) -> None:
         object_data = self.get_object_data(payload)
         name = object_data["name"]
@@ -206,9 +169,6 @@ class BaremetalEvents:
         netbox.set_state.delay(name, None, "introspection")
         netbox.set_state.delay(name, None, "deployment")
 
-        # system should be in state a
-        netbox.connect.delay(name, "a")
-
         # remove internal flavor
         openstack.baremetal_delete_internal_flavor.delay(name)
 

osism/settings.py

@@ -27,9 +27,6 @@ NETBOX_URL = os.getenv("NETBOX_API")
 NETBOX_TOKEN = os.getenv("NETBOX_TOKEN", read_secret("NETBOX_TOKEN"))
 IGNORE_SSL_ERRORS = os.getenv("IGNORE_SSL_ERRORS", "True") == "True"
 
-BASE_PATH = os.getenv("BASE_PATH", "/devicetype-library/device-types/")
-VENDORS = os.getenv("VENDORS", "").split()
-
 # 43200 seconds = 12 hours
 GATHER_FACTS_SCHEDULE = float(os.getenv("GATHER_FACTS_SCHEDULE", "43200.0"))
 INVENTORY_RECONCILER_SCHEDULE = float(

osism/tasks/__init__.py

@@ -160,20 +160,6 @@ def run_ansible_in_environment(
             env=env,
         )
 
-    # execute local netbox playbooks
-    elif worker == "osism-ansible" and environment == "netbox-local":
-        if locking:
-            lock.acquire()
-
-        command = f"/run-{environment}.sh {role} {joined_arguments}"
-        logger.info(f"RUN {command}")
-        p = subprocess.Popen(
-            command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            shell=True,
-        )
-
     # execute all other roles
     else:
         if locking:
@@ -207,6 +193,50 @@ def run_ansible_in_environment(
     return result
 
 
+def run_command(
+    request_id,
+    command,
+    env,
+    *arguments,
+    publish=True,
+    locking=False,
+    auto_release_time=3600,
+):
+    result = ""
+    command_env = os.environ.copy()
+    command_env.update(env)
+
+    if locking:
+        lock = Redlock(
+            key=f"lock-{command}",
+            masters={redis},
+            auto_release_time=auto_release_time,
+        )
+
+    p = subprocess.Popen(
+        [command] + list(arguments),
+        env=command_env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    while p.poll() is None:
+        line = p.stdout.readline().decode("utf-8")
+        if publish:
+            redis.xadd(request_id, {"type": "stdout", "content": line})
+        result += line
+
+    rc = p.wait(timeout=60)
+
+    if publish:
+        redis.xadd(request_id, {"type": "rc", "content": rc})
+        redis.xadd(request_id, {"type": "action", "content": "quit"})
+
+    if locking:
+        lock.release()
+
+    return result
+
+
 def handle_task(t, wait=True, format="log", timeout=3600):
     global redis
 

osism/tasks/ansible.py

@@ -1,25 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
-import functools
-from threading import RLock
-
 from celery import Celery
-import kombu.utils
 
 from osism import settings
 from osism.tasks import Config, run_ansible_in_environment
 
-# https://github.com/celery/kombu/issues/1804
-if not getattr(kombu.utils.cached_property, "lock", None):
-    setattr(
-        kombu.utils.cached_property,
-        "lock",
-        functools.cached_property(lambda _: RLock()),
-    )
-    # Must call __set_name__ here since this cached property is not defined in the context of a class
-    # Refer to https://docs.python.org/3/reference/datamodel.html#object.__set_name__
-    kombu.utils.cached_property.lock.__set_name__(kombu.utils.cached_property, "lock")
-
 app = Celery("ansible")
 app.config_from_object(Config)