nornir-collection 0.0.1__py3-none-any.whl

nornir_collection/netbox/verify_device_primary_ip.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""
+This module load the NetBox device inventory, find active management IP addresses, and verify their integrity.
+The Main function is intended to import and execute by other scripts.
+"""
+
+
+import sys
+import subprocess  # nosec
+from nornir_collection.netbox.utils import get_nb_resources
+from nornir_collection.utils import (
+    print_task_title,
+    task_name,
+    exit_error,
+    load_yaml_file,
+    task_result,
+)
+
+
+__author__ = "Willi Kubny"
+__maintainer__ = "Willi Kubny"
+__version__ = "1.0"
+__license__ = "MIT"
+__email__ = "willi.kubny@dreyfusbank.ch"
+__status__ = "Production"
+
+
+def main(  # pylint: disable=dangerous-default-value,too-many-arguments,too-many-statements
+    nr_config: str,
+    mgmt_subnets: list[str],
+    add_addresses: list[str] = [],
+    exclude_addresses: list[str] = [],
+) -> None:
+    """
+    Main function is intended to import and execute by other scripts.
+    It verify the integrity of NetBox primary device management IP addresses.
+
+    * Args:
+        * nr_config (str): Path to the Nornir configuration file.
+        * mgmt_subnets (list[str]): List of management subnets to scan for active IP addresses.
+        * add_addresses (list[str], optional): List of additional IP addresses to scan. Defaults to [].
+        * exclude_addresses (list[str], optional): List of IP addresses to exclude. Defaults to [].
+
+    * Steps:
+        * Loads the NetBox device inventory.
+        * Finds all active NetBox primary device management IP addresses.
+        * Scans all provided management subnets and additional addresses for active IP addresses using fping.
+        * Verifies the integrity of NetBox primary device management IP addresses by checking:
+            * If any NetBox primary IP addresses are missing from the scan.
+            * If any active IP addresses from the scan are not part of the NetBox inventory.
+
+    * Exits:
+        * Exits with code 1 if the Nornir configuration file is empty or if any task fails.
+    """
+    # pylint: disable=invalid-name,too-many-locals
+
+    #### Load NetBox Device Inventory ########################################################################
+
+    task_title = "Load NetBox Device Inventory"
+    print_task_title(title=task_title)
+
+    # Load the Nornir yaml config file as dict and print a error message
+    nr_config_dict = load_yaml_file(
+        file=nr_config, text="Load Nornir Config File", silent=False, verbose=False
+    )
+    # Check the loaded config file and exit the script with exit code 1 if the dict is empty
+    if not nr_config_dict:
+        sys.exit(1)
+
+    task_text = "Load NetBox Device Inventory"
+    print(task_name(text=task_text))
+
+    # Get the NetBox URL (Authentication token will be loaded as nb_token env variable)
+    nb_url = nr_config_dict["inventory"]["options"]["nb_url"]
+    params = {"status": "active", "has_primary_ip": True}
+    nb_devices = get_nb_resources(url=f"{nb_url}/api/dcim/devices/?limit=0", params=params)
+
+    # Set the task level to INFO if the nb_devices list if not empty
+    level_name = "INFO" if nb_devices else "ERROR"
+    success = "True" if level_name == "INFO" else "False"
+    print(task_result(text=task_text, changed=False, level_name=level_name))
+    print(
+        f"'Load NetBox device inventory' -> NetBoxResult <Success: {success}>\n"
+        + f"-> NetBox device count: {len(nb_devices)}",
+    )
+    # Exit the script if fping failed and the nb_devices list is empty
+    if not nb_devices:
+        exit_error(task_text=f"{task_text} Failed", msg="-> No NetBox device in API response")
+
+    #### Find all NetBox primary device MGMT ip-addresses ###################################################
+
+    task_text = "Find all active NetBox primary device mgmt ip-addresses"
+    print(task_name(text=task_text))
+
+    # Loop over all NetBox devices and create a list of their primary ipv4 address
+    nb_mgmt_ips = [str(device["primary_ip"]["address"])[:-3] for device in nb_devices]
+
+    # Set the task level to INFO if the nb_mgmt_ips list if not empty
+    level_name = "INFO" if nb_mgmt_ips else "ERROR"
+    success = "True" if level_name == "INFO" else "False"
+    print(task_result(text=task_text, changed=False, level_name=level_name))
+    print(
+        f"'Find active mgmt ip-addresses' -> NornirResult <Success: {success}>\n"
+        + f"-> NetBox primary device mgmt ip-address count: {len(nb_mgmt_ips)}",
+    )
+    # Exit the script if fping failed and the nb_mgmt_ips list is empty
+    if not nb_mgmt_ips:
+        exit_error(task_text=f"{task_text} Failed", msg="-> No NetBox primary device mgmt ip-addresses found")
+
+    #### Find all active MGMT ip-addresses with fping #######################################################
+
+    task_title = "Scan all script input mgmt ip-addresses with fping"
+    print_task_title(title=task_title)
+
+    task_text = "Find all active mgmt ip-addresses with fping"
+    print(task_name(text=task_text))
+
+    # Loop over all mgmt_subnets and create a list of active ip-addresses
+    alive_ips = []
+    for subnet in mgmt_subnets:
+        # fmt: off
+        fping = subprocess.run(["fping", "-a", "-g", subnet,], check=False, capture_output=True) # nosec
+        # fmt: on
+        output = fping.stdout.decode("utf-8").splitlines()
+        alive_ips.extend([ip for ip in output if ip not in exclude_addresses])
+    # Loop over all add_addresses and add them to the list of active ip-addresses
+    for ip in add_addresses:
+        # fmt: off
+        fping = subprocess.run(["fping", "-a", ip,], check=False, capture_output=True) # nosec
+        # fmt: on
+        output = fping.stdout.decode("utf-8").splitlines()
+        alive_ips.extend([ip for ip in output if ip not in exclude_addresses])
+
+    # Set the task level to INFO if the alive_ip list if not empty
+    level_name = "INFO" if alive_ips else "ERROR"
+    success = "True" if level_name == "INFO" else "False"
+    print(task_result(text=task_text, changed=False, level_name=level_name))
+    print(
+        f"'Find active mgmt ip-addresses' -> NornirResult <Success: {success}>\n"
+        + f"-> Active mgmt ip-address count: {len(alive_ips)}",
+    )
+    # Exit the script if fping failed and the alive_ips list is empty
+    if not alive_ips:
+        exit_error(
+            task_text=f"{task_text} Failed",
+            msg=["-> No active mgmt ip-addresses found", "-> Verify the provided 'mgmt_subnets' input list"],
+        )
+
+    #### Verify NetBox primary device MGMT ip-addresses integrity ###########################################
+
+    # Set a boolean to True if the integrity check fails
+    integrity_failed = False
+
+    task_title = "Verify NetBox primary device mgmt ip-address integrity"
+    print_task_title(title=task_title)
+
+    #### Verify NetBox ip-addresses not part of the script input parameters
+
+    task_text = "Verify missing NetBox device ip-addresses for scan"
+    print(task_name(text=task_text))
+
+    # Find all NetBox primary ip-addresses which are not covered by the script input parameters
+    ip_diff = list(set(nb_mgmt_ips) - set(alive_ips))
+
+    # Set the task level to INFO if the ip_diff list if not empty
+    level_name = "ERROR" if ip_diff else "INFO"
+    success = "True" if level_name == "INFO" else "False"
+    print(task_result(text=task_text, changed=False, level_name=level_name))
+    print(f"'{task_text}' -> NornirResult <Success: {success}>")
+    if ip_diff:
+        integrity_failed = True
+        print("-> The following NetBox device ip-addresses are not part of the script input parameters")
+        for ip in ip_diff:
+            print(f"- {ip}")
+    else:
+        print("-> All NetBox device ip-addresses are part of the script input parameters")
+
+    #### Verify NetBox primary device mgmt ip-address integrity
+
+    task_text = "Verify active device ip-addresses covered by NetBox"
+    print(task_name(text=task_text))
+
+    # Find all active ip-addresses which are not part of a NetBox primary ip-address
+    ip_diff = list(set(alive_ips) - set(nb_mgmt_ips))
+
+    # Set the task level to INFO if the ip_diff list if not empty
+    level_name = "ERROR" if ip_diff else "INFO"
+    success = "True" if level_name == "INFO" else "False"
+    print(task_result(text=task_text, changed=False, level_name=level_name))
+    print(f"'{task_text}' -> NornirResult <Success: {success}>")
+    if ip_diff:
+        integrity_failed = True
+        print("-> The following active device ip-addresses are not part of the NetBox inventory")
+        for ip in ip_diff:
+            print(f"- {ip}")
+    else:
+        print("-> All active device ip-addresses are part of the NetBox inventory")
+
+    # Exit the script with an error if the integrity check failed
+    if integrity_failed:
+        print("\r")
+        exit_error(task_text=f"{task_title} Failed", msg="-> Verify the script results for failes tasks")

nornir_collection/nornir_plugins/inventory/netbox.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""
+This module contains the Nornir inventory plugin DSCNetBoxInventory.
+
+The functions are ordered as followed:
+- Helper Functions
+- Inventory Plugin
+"""
+
+import os
+import logging
+from typing import Any, Dict, List, Optional, Union, Type
+import requests
+import ruamel.yaml
+from nornir.core.inventory import (
+    ConnectionOptions,
+    Defaults,
+    Group,
+    Groups,
+    Host,
+    HostOrGroup,
+    Hosts,
+    Inventory,
+    ParentGroups,
+)
+from nornir_collection.utils import print_task_name, task_info
+
+logger = logging.getLogger(__name__)
+
+
+#### Helper Functions #######################################################################################
+
+
+def _get_connection_options(data: Dict[str, Any]) -> Dict[str, ConnectionOptions]:
+    """
+    TBD
+    """
+    # pylint: disable=invalid-name
+    cp = {}
+    for cn, c in data.items():
+        cp[cn] = ConnectionOptions(
+            hostname=c.get("hostname"),
+            port=c.get("port"),
+            username=c.get("username"),
+            password=c.get("password"),
+            platform=c.get("platform"),
+            extras=c.get("extras"),
+        )
+    return cp
+
+
+def _get_defaults(data: Dict[str, Any]) -> Defaults:
+    """
+    TBD
+    """
+    return Defaults(
+        hostname=data.get("hostname"),
+        port=data.get("port"),
+        username=data.get("username"),
+        password=data.get("password"),
+        platform=data.get("platform"),
+        data=data.get("data"),
+        connection_options=_get_connection_options(data.get("connection_options", {})),
+    )
+
+
+def _get_inventory_element(
+    typ: Type[HostOrGroup], data: Dict[str, Any], name: str, defaults: Defaults
+) -> HostOrGroup:
+    """
+    TBD
+    """
+    return typ(
+        name=name,
+        hostname=data.get("hostname"),
+        port=data.get("port"),
+        username=data.get("username"),
+        password=data.get("password"),
+        platform=data.get("platform"),
+        data=data.get("data"),
+        groups=data.get("groups"),  # this is a hack, we will convert it later to the correct type
+        defaults=defaults,
+        connection_options=_get_connection_options(data.get("connection_options", {})),
+    )
+
+
+#### Inventory Plugin #######################################################################################
+
+
+class DSCNetBoxInventory:
+    """
+    This class is a deviation of the Nornir plugin NetBoxInventory2.
+    This plugin have to be registered since Nornir 3.0
+
+    Inventory plugin that uses `NetBox <https://github.com/netbox-community/netbox>`_ as backend.
+    Note:
+        Additional data provided by the NetBox devices API endpoint will be
+        available through the NetBox Host data attribute.
+    Environment Variables:
+        * ``NB_URL``: Corresponds to nb_url argument
+        * ``NB_TOKEN``: Corresponds to nb_token argument
+    Arguments:
+        nb_url: NetBox url
+        nb_token: NetBox API token
+        ssl_verify: Enable/disable certificate validation
+            (defaults to True)
+    """
+
+    # pylint: disable=invalid-name
+
+    def __init__(  # pylint: disable=unused-argument,too-many-arguments
+        self,
+        nb_url: Optional[str] = None,
+        nb_token: Optional[str] = None,
+        ssl_verify: Union[bool, str] = True,
+        **kwargs: Any,
+    ) -> None:
+        """
+        Init function
+        """
+        nb_url = nb_url or os.environ.get("NB_URL")
+        nb_token = nb_token or os.environ.get("NB_TOKEN")
+
+        self.task_text = "NORNIR initialize inventory plugin DSCNetBoxInventory"
+        self.nb_url = nb_url
+        self.session = requests.Session()
+        self.session.headers.update({"Authorization": f"Token {nb_token}"})
+        self.session.verify = ssl_verify
+
+    def _get_resources(  # pylint: disable=dangerous-default-value
+        self, url: str, params: Dict[str, Any] = {}
+    ) -> List[Dict[str, Any]]:
+        """
+        TBD
+        """
+        # Define the resource list
+        resources: List[Dict[str, Any]] = []
+        # While there is a next page continue the loop
+        while url:
+            # Do the http request
+            r = self.session.get(url, params=params)
+            # Verify the response code
+            if not r.status_code == 200:
+                raise ValueError(f"Failed to get data from NetBox instance {self.nb_url}")
+            # Extract the json data from the http response
+            resp = r.json()
+            # Add the response to the resource list
+            resources.extend(resp.get("results"))
+            # Get the url of the next page
+            url = resp.get("next")
+
+        # Retrun the resources list
+        return resources
+
+    def load(self) -> Inventory:
+        """
+        TBD
+        """
+        # pylint: disable=too-many-statements,too-many-branches,too-many-locals
+        yml = ruamel.yaml.YAML(typ="safe")
+
+        print_task_name(text=self.task_text)
+
+        # Load all NetBox devices with a primary IP address assigned
+        devices: List[Dict[str, Any]] = []
+        devices.extend(
+            self._get_resources(
+                url=f"{self.nb_url}/api/dcim/devices/",
+                params={"limit": 1000, "has_primary_ip": True},
+            )
+        )
+
+        # Create the defaults
+        defaults = Defaults()
+        defaults_dict: Dict[str, Any] = {}
+        with open("inventory/defaults.yaml", "r", encoding="utf-8") as f:
+            defaults_dict = yml.load(f) or {}
+        defaults = _get_defaults(defaults_dict)
+
+        # Create the hosts
+        hosts = Hosts()
+        for device in devices:
+            serialized_device: Dict[Any, Any] = {}
+            serialized_device["data"] = device
+
+            # Remove the custom_fields key and make the custom_field native to Nornir
+            for cf, value in device["custom_fields"].items():
+                serialized_device["data"][cf] = value
+            serialized_device["data"].pop("custom_fields")
+
+            # Remove the config_context key and make the config_context native to Nornir
+            for cc, value in device["config_context"].items():
+                serialized_device["data"][cc] = value
+            serialized_device["data"].pop("config_context")
+
+            # Set the Nornir hostname (ip address or fqdn)
+            hostname = None
+            if device.get("primary_ip"):
+                hostname = device.get("primary_ip", {}).get("address", "").split("/")[0]
+            else:
+                if device.get("name") is not None:
+                    hostname = device["name"]
+            serialized_device["hostname"] = hostname
+
+            # Set the Nornir host name
+            if serialized_device["data"].get("virtual_chassis"):
+                name = serialized_device["data"].get("virtual_chassis").get("name")
+            else:
+                name = serialized_device["data"].get("name") or str(serialized_device["data"].get("id"))
+
+            # Add virtual chassis master serial to the virtual_chassis dict
+            if serialized_device["data"].get("virtual_chassis"):
+                master_serial = device["serial"] if device["serial"] else None
+                serialized_device["data"]["virtual_chassis"]["master"]["serial"] = master_serial
+
+            # Flatten the list of Tags which contains only the slug of each NetBox tag
+            device["tags"] = [tag["slug"] for tag in device["tags"]]
+
+            # Extract the device connection options
+            # Add Nornir connection options from the device context data
+            if "connection_options" in device and device["connection_options"] is not None:
+                serialized_device["connection_options"] = device["connection_options"]
+            else:
+                serialized_device["connection_options"] = {}
+            # Remove the connection_options key from the serialized_device data
+            if (
+                "connection_options" in serialized_device["data"]
+                and serialized_device["data"]["connection_options"] is not None
+            ):
+                serialized_device["data"].pop("connection_options")
+
+            # Set the Nornir host elements
+            hosts[name] = _get_inventory_element(Host, serialized_device, name, defaults)
+
+        # Create the groups
+        groups = Groups()
+        groups_dict: Dict[str, Any] = {}
+        for n, g in groups_dict.items():
+            groups[n] = _get_inventory_element(Group, g, n, defaults)
+        for g in groups.values():
+            g.groups = ParentGroups([groups[g] for g in g.groups])
+        for h in hosts.values():
+            h.groups = ParentGroups([groups[g] for g in h.groups])
+
+        # Print the DSCNetBoxInventory result
+        print(task_info(text=self.task_text, changed=False))
+        print(f"'{self.task_text}' -> NornirResponse <Success: True>")
+        print(f"-> Loaded NetBox base inventory data of {len(hosts)} devices")
+
+        return Inventory(hosts=hosts, groups=groups, defaults=defaults)

nornir_collection/nornir_plugins/inventory/staggered_yaml.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""
+This module contains Nornir inventory plugin StaggeredYamlInventory.
+
+The functions are ordered as followed:
+- Helper Functions
+- Inventory Plugin
+"""
+
+import pathlib
+import glob
+from typing import Literal
+import ruamel.yaml
+from nornir.core.inventory import Inventory, Group, Groups, Host, Hosts, Defaults, ParentGroups
+from nornir.plugins.inventory.simple import _get_defaults, _get_inventory_element
+from nornir_collection.utils import (
+    print_task_name,
+    task_info,
+    task_error,
+    exit_error,
+    load_multiple_yaml_files_to_string,
+)
+
+
+#### Inventory Plugin #######################################################################################
+
+
+class StaggeredYamlInventory:
+    """
+    This class is a deviation of the Nornir builtin SimpleInventory.
+    This plugin have to be registered since Nornir 3.0
+    """
+
+    # pylint: disable=invalid-name
+
+    def __init__(
+        self,
+        hosts_base: str = "hosts.yaml",
+        groups_base: str = "groups.yaml",
+        defaults_base: str = "defaults.yaml",
+        encoding: str = "utf-8",
+    ) -> None:
+        """
+        StaggeredYamlInventory is an inventory plugin that loads data from multiple yaml files. The the base
+        path all yaml files starting with hosts, groups, or defaults will be concatenated and loaded as dict.
+        The yaml files follow the same structure as the native objects
+        Args:
+          hosts_base: base path recursively to files with hosts definition
+          groups_base: basepath recursively to files with groups definition.
+                If it doesn't exist it will be skipped
+          defaults_base: base path recursively to files with defaults definition.
+                If it doesn't exist it will be skipped
+          encoding: Encoding used to save inventory files. Defaults to utf-8
+        """
+
+        self.task_text = "NORNIR initialize inventory plugin StaggeredYamlInventory"
+        # Get the base path for the hosts, groups and defaults filed
+        self.hosts_base = pathlib.Path(hosts_base).expanduser()
+        self.groups_base = pathlib.Path(groups_base).expanduser()
+        self.defaults_base = pathlib.Path(defaults_base).expanduser()
+        # Create lists with all files recursively which starts with hosts, groups or defaults
+        self.hosts_files = glob.glob(str(self.hosts_base) + "/**/hosts*.yaml", recursive=True)
+        self.groups_files = glob.glob(str(self.groups_base) + "/**/groups*.yaml", recursive=True)
+        self.defaults_files = glob.glob(str(self.defaults_base) + "/**/defaults*.yaml", recursive=True)
+        self.encoding = encoding
+
+    def load_multiple_yaml_files(self, inv_type: Literal["hosts", "groups", "defaults"]) -> dict:
+        """
+        TBD
+        """
+        yml = ruamel.yaml.YAML(typ="safe")
+        yaml_string = str()
+
+        # Use mapping to avoid long if elif else statements
+        file_list = {"hosts": self.hosts_files, "groups": self.groups_files, "defaults": self.defaults_files}
+        # Load the yaml files from file_list and concatinate them to one string
+        yaml_string = load_multiple_yaml_files_to_string(file_list=file_list[inv_type], silent=True)
+
+        try:
+            yaml_dict = yml.load(yaml_string) or {}
+
+        except ruamel.yaml.constructor.DuplicateKeyError as error:
+            print(task_error(text=self.task_text, changed=False))
+            print(f"'Load {inv_type} inventory yaml files' -> NornirResponse <Success: False>")
+            print("-> Duplicate key in yaml files", "\n\n", error)
+            # Exit the script with a proper message
+            exit_error(
+                task_text=self.task_text,
+                text=f"ALERT: LOAD {inv_type.upper()} INVENTORY YAML FILES FAILED!",
+                msg=f"-> Analyse the {inv_type} inventory yaml files and identify the root cause",
+            )
+
+        return yaml_dict
+
+    def load(self) -> Inventory:
+        """
+        TBD
+        """
+        print_task_name(text=self.task_text)
+
+        if self.defaults_files:
+            defaults_dict = self.load_multiple_yaml_files(inv_type="defaults")
+
+            defaults = _get_defaults(defaults_dict)
+        else:
+            defaults = Defaults()
+
+        hosts = Hosts()
+        hosts_dict = self.load_multiple_yaml_files(inv_type="hosts")
+
+        for n, h in hosts_dict.items():
+            hosts[n] = _get_inventory_element(Host, h, n, defaults)
+
+        groups = Groups()
+        if self.groups_files:
+            groups_dict = self.load_multiple_yaml_files(inv_type="groups")
+
+            for n, g in groups_dict.items():
+                groups[n] = _get_inventory_element(Group, g, n, defaults)
+
+            for g in groups.values():
+                g.groups = ParentGroups([groups[g] for g in g.groups])
+
+        try:
+            for h in hosts.values():
+                h.groups = ParentGroups([groups[g] for g in h.groups])
+
+        except KeyError as error:
+            print(task_error(text=self.task_text, changed=False))
+            print("'Assign groups to hosts' -> NornirResponse <Success: False>")
+            print(f"-> Group {error} not found")
+            # Exit the script with a proper message
+            exit_error(
+                task_text=self.task_text,
+                text="ALERT: ASSIGN GROUPS TO HOSTS FAILED!",
+                msg="-> Analyse the Nornir group files",
+            )
+
+        print(task_info(text=self.task_text, changed=False))
+        print(f"'{self.task_text}' -> NornirResponse <Success: True>")
+        print("-> Loaded YAML inventory data")
+
+        return Inventory(hosts=hosts, groups=groups, defaults=defaults)