nornir-collection 0.0.10__tar.gz → 0.0.12__tar.gz

{nornir_collection-0.0.10 → nornir_collection-0.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nornir-collection
-Version: 0.0.10
+Version: 0.0.12
 Summary: Nornir-Collection contains network automation functions and complete IaC workflows with         Nornir and other python libraries. It contains Nornir tasks and general functions in Nornir style.
 Author: Willi Kubny
 Author-email: willi.kubny@gmail.ch
@@ -12,7 +12,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: nornir==3.3.0
 Requires-Dist: nornir-jinja2==0.2.0
-Requires-Dist: nornir-salt==0.20.3
+Requires-Dist: nornir-salt==0.22.2
 Requires-Dist: nornir-netmiko==1.0.1
 Requires-Dist: nornir-scrapli==2023.7.30
 Requires-Dist: scrapli-netconf==2023.7.30
@@ -20,6 +20,7 @@ Requires-Dist: scrapli[ssh2]
 Requires-Dist: scrapli[genie]
 Requires-Dist: nornir-utils==0.2.0
 Requires-Dist: pynetbox==7.4.0
+Requires-Dist: ipfabric==7.0.2
 Requires-Dist: py-pure-client==1.51.0
 Requires-Dist: nornir-pyfgt==1.0.3
 Requires-Dist: python-nmap==0.7.1

{nornir_collection-0.0.10 → nornir_collection-0.0.12}/nornir_collection/cisco/configuration_management/utils.py

@@ -373,7 +373,7 @@ def init_args_for_netconf_cm() -> argparse.Namespace:
         "--pre-check",
         action="store_true",
         default=False,
-        help="disable the pre-configuration check (default: False)",
+        help="enable the pre-configuration check (default: False)",
     )
 
     # Verify the provided arguments and print the custom argparse error message in case any error or wrong

nornir_collection-0.0.10/nornir_collection/netbox/scan_prefixes_and_update_ip_addresses.py → nornir_collection-0.0.12/nornir_collection/netbox/update_prefixes_ip_addresses.py

@@ -5,6 +5,7 @@ IP address, and vlan information.
 The Main function is intended to import and execute by other scripts.
 """
 
+import os
 import sys
 import ipaddress
 import urllib.parse
@@ -12,6 +13,7 @@ from typing import Callable, Literal, Union
 from concurrent.futures import ThreadPoolExecutor
 import requests
 import nmap
+from ipfabric import IPFClient
 from nornir_collection.netbox.utils import (
     get_nb_resources,
     post_nb_resources,
@@ -114,6 +116,7 @@ def create_nb_response_result(
     data: Union[dict, list],
     task_text: str,
     text: str,
+    ds: Literal["nmap", "ip-fabric"] = None,
 ) -> tuple:
     """
     Verify the NetBox response and return the result.
@@ -143,14 +146,19 @@ def create_nb_response_result(
         # Create a list of fields that have been updated
         updated_fields = []
 
-        # If data is a list of ip addresses from a active scanned prefix
+        # If data is a list of ip addresses from a active scanned prefix of IP-Fabric
         if nb_type in "ip" and isinstance(data, list):
             # Create a list of ip addresses that have been updated
             for ip in data:
                 address = ip["address"]
                 dns_name = ip["dns_name"] if "dns_name" in ip and ip["dns_name"] else "None"
-                ports = ", ".join([str(p) for p in ip["ports"]]) if "ports" in ip and ip["ports"] else "None"
-                updated_fields.append(f"- {address} (DNS: {dns_name}, Ports: {ports})")
+                if ds == "nmap":
+                    ports = (
+                        ", ".join([str(p) for p in ip["ports"]]) if "ports" in ip and ip["ports"] else "None"
+                    )
+                    updated_fields.append(f"- {address} (DNS: {dns_name}, Ports: {ports})")
+                else:
+                    updated_fields.append(f"- {address} (DNS: {dns_name})")
         # If data is a dictionary containing the prefix or information vlan associated with a prefix
         elif nb_type in ("ip", "vlan") and isinstance(data, dict):
             # If the response json contains the key 'vid' the response is from a vlan and has no VRF
@@ -195,7 +203,9 @@ def create_nb_response_result(
     return result, True
 
 
-def create_nb_ip_payload(parent_prefix: dict, data: list, desired_status: str = None) -> dict:
+def create_nb_ip_payload(
+    parent_prefix: dict, data: list, ds: Literal["nmap", "ip-fabric"] = None, desired_status: str = None
+) -> dict:
     """
     Create a NetBox REST API payload.
     To add or delete IP addresses of an active scanned prefix or update the following fields:
@@ -220,22 +230,25 @@ def create_nb_ip_payload(parent_prefix: dict, data: list, desired_status: str =
         # Add the 'id' if it exists (not needed for post requests)
         if "id" in ip:
             item["id"] = ip["id"]
-        # If desired_status is not None, the payload is for active scanned ip addresses
+        # If desired_status is not None, the payload is for active scanned ip addresses or IP-Fabric
         if desired_status:
             # Add the 'address' and 'status' to the payload
             item["address"] = ip["address"]
             item["status"] = desired_status
-            # If the ip address exists in the nmap_ips list
-            nmap_ip = [x for x in parent_prefix["nmap_ips"] if x["address"] == ip["address"]]
+            # If the ip address exists in the datasource_ips list
+            datasource_ip = [x for x in parent_prefix["datasource_ips"] if x["address"] == ip["address"]]
             # Add the 'dns_name'
-            item["dns_name"] = nmap_ip[0]["dns_name"] if nmap_ip else ""
-            # Format the 'ports' as a Markdown table and add the 'ports'
-            if nmap_ip and nmap_ip[0]["ports"]:
-                md = [["Port", "State", "Service"]]
-                md.extend([[f"{k}/tcp", v["state"], v["name"]] for k, v in nmap_ip[0]["ports"].items()])
-                item["custom_fields"] = {"ipaddress_ports": make_markdown_table(md)}
-            else:
-                item["custom_fields"] = {"ipaddress_ports": None}
+            item["dns_name"] = datasource_ip[0]["dns_name"] if datasource_ip else ""
+            # If it's a nmap scan format the 'ports' as a Markdown table and add the 'ports'
+            if ds == "nmap":
+                if datasource_ip and datasource_ip[0]["ports"]:
+                    md = [["Port", "State", "Service"]]
+                    md.extend(
+                        [[f"{k}/tcp", v["state"], v["name"]] for k, v in datasource_ip[0]["ports"].items()]
+                    )
+                    item["custom_fields"] = {"ipaddress_ports": make_markdown_table(md)}
+                else:
+                    item["custom_fields"] = {"ipaddress_ports": None}
         # If the desired_status is None, the payload is for all ip addresses of a prefix
         else:
             # Add the 'vrf' to the payload
@@ -253,23 +266,10 @@ def create_nb_ip_payload(parent_prefix: dict, data: list, desired_status: str =
     return payload
 
 
-def get_netbox_ip_addresses_and_scan_prefix(nb_url: str, prefix: dict) -> tuple:
+def nmap_scan_prefix(prefix: dict, result: list) -> tuple:
     """
-    Get NetBox IP addresses and scan a prefix with nmap.
-    This function can be used within a ThreadPoolExecutor.
-
-    Args:
-        nb_url (str): The URL of the NetBox instance.
-        prefix (dict): The prefix to scan and retrieve IP addresses for.
-
-    Returns:
-        tuple: A tuple containing the result list and the updated prefix dictionary.
+    TBD
     """
-    # Create list to collect the results
-    result = []
-
-    #### Scan the prefix with nmap ###########################################################################
-
     # Get the prefix length from the prefix
     prefixlen = ipaddress.ip_network(prefix["prefix"]).prefixlen
     # Get the network and broadcast address of the prefix to exclude them from the nmap scan
@@ -282,7 +282,7 @@ def get_netbox_ip_addresses_and_scan_prefix(nb_url: str, prefix: dict) -> tuple:
     nm = nmap.PortScanner()
     nm.scan(hosts=prefix["prefix"], arguments=arguments, sudo=True)
     if nm.all_hosts():
-        prefix["nmap_ips"] = [
+        prefix["datasource_ips"] = [
             {
                 "address": f"{nm[host]['addresses']['ipv4']}/{prefixlen}",
                 "dns_name": nm[host]["hostnames"][0]["name"],
@@ -291,16 +291,96 @@ def get_netbox_ip_addresses_and_scan_prefix(nb_url: str, prefix: dict) -> tuple:
             for host in nm.all_hosts()
         ]
     else:
-        prefix["nmap_ips"] = []
+        prefix["datasource_ips"] = []
 
     # Print the task result
     text = f"Nmap Scan Prefix {prefix['prefix']} for active IP-Addresses"
     result.append(
         f"{task_result(text=text, changed=False, level_name='INFO')}\n"
         + f"'{text}' -> NetBoxResponse <Success: True>\n"
-        + f"-> Nmap prefix scan ip-address count: {len(prefix['nmap_ips'])}",
+        + f"-> Nmap prefix scan ip-address count: {len(prefix['datasource_ips'])}"
     )
 
+    return result, prefix
+
+
+def get_ipfabric_data_for_prefix(prefix: dict, result: list) -> tuple:
+    """
+    TBD
+    """
+    # Connect to IP-Fabric
+    ipf = IPFClient(
+        base_url=os.environ["IPF_URL"], auth=os.environ["IPF_TOKEN"], snapshot_id="$last", verify=False
+    )
+
+    # Get the prefix length from the prefix
+    prefixlen = ipaddress.ip_network(prefix["prefix"]).prefixlen
+    # Get all ip-addresses of the prefix from the IP-Fabric technology arp table
+    # Remove duplicate ip-addresses as the arp table can contain multiple entries for the same ip-address
+    filters = {"ip": ["cidr", prefix["prefix"]]}
+    all_ips = list(set([x["ip"] for x in ipf.technology.addressing.arp_table.all(filters=filters)]))
+    # Get all ip-addresses of the prefix from the IP-Fabric inventory hosts table
+    # Set the ip-address as key to access the dict easier later
+    columns = ["ip", "dnsName"]
+    host_ips = {
+        x["ip"]: x["dnsName"] or "" for x in ipf.inventory.hosts.all(filters=filters, columns=columns)
+    }
+    # Add a list of all ip-addresses and other details to prefix
+    prefix["datasource_ips"] = [
+        {
+            "address": f"{ip}/{prefixlen}",
+            "dns_name": host_ips[ip] if ip in host_ips.keys() else "",
+            "ports": {},
+        }
+        for ip in all_ips
+    ]
+
+    # Print the task result
+    text = f"IP-Fabric Get Data for Prefix {prefix['prefix']} IP-Addresses"
+    result.append(
+        f"{task_result(text=text, changed=False, level_name='INFO')}\n"
+        + f"'{text}' -> IPFResponse <Success: True>\n"
+        + f"-> IP-Fabric prefix ip-address count: {len(prefix['datasource_ips'])}"
+    )
+
+    return result, prefix
+
+
+def get_nb_ips_and_external_datasource(nb_url: str, prefix: dict, ds: Literal["nmap", "ip-fabric"]) -> tuple:
+    """
+    Get NetBox IP addresses and scan a prefix with nmap or get the data from IP-Fabric.
+    This function can be used within a ThreadPoolExecutor.
+
+    Args:
+        nb_url (str): The URL of the NetBox instance.
+        prefix (dict): The prefix to scan and retrieve IP addresses for.
+        ds (Literal["nmap", "ip-fabric"]): The datasource to use for the IP addresses.
+
+    Returns:
+        tuple: A tuple containing the result list and the updated prefix dictionary.
+    """
+    # Create list to collect the results
+    failed = False
+    result = []
+
+    # Get the ip-addresses of the prefix from the datasource
+    if ds == "nmap":
+        # Scan the prefix with nmap
+        result, prefix = nmap_scan_prefix(prefix=prefix, result=result)
+    elif ds == "ip-fabric":
+        # Get the ip-addresses from the IP-Fabric
+        result, prefix = get_ipfabric_data_for_prefix(prefix=prefix, result=result)
+    else:
+        # Invalid datasource
+        failed = True
+        text = f"Get NetBox IP-Addresses of Prefix {prefix['prefix']}"
+        result.append(
+            f"{task_result(text=text, changed=False, level_name='ERROR')}\n"
+            + f"'{text}' -> NetBoxResponse <Success: True>\n"
+            + f"-> Invalid Datasource '{ds}'\n"
+        )
+        return result, prefix, failed
+
     #### Get NetBox ip-addresses ############################################################################
 
     # Add a list of dicts (id & address) of all NetBox ip-addresses for prefix
@@ -346,7 +426,7 @@ def get_netbox_ip_addresses_and_scan_prefix(nb_url: str, prefix: dict) -> tuple:
         + f"-> Deprecated ip-address count: {len(prefix['deprecated_ips'])}",
     )
 
-    return result, prefix
+    return result, prefix, failed
 
 
 def create_ip_list(loop_list: list[dict], check_list: list[dict], is_in_both: bool) -> list[dict]:
@@ -379,7 +459,7 @@ def create_ip_list(loop_list: list[dict], check_list: list[dict], is_in_both: bo
     return ip_list
 
 
-def update_discovered_ip_addresses(nb_url: str, prefix: dict) -> tuple:
+def update_discovered_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap", "ip-fabric"]) -> tuple:
     """
     Posts new auto-discovered IP addresses to NetBox.
 
@@ -396,12 +476,16 @@ def update_discovered_ip_addresses(nb_url: str, prefix: dict) -> tuple:
     task_text = "Add Auto-Discovered IP-Addresses"
 
     # Add nmap scan ip-addresses that are not in the existing ip-addresses dict
-    add_ips = create_ip_list(loop_list=prefix["nmap_ips"], check_list=prefix["all_ips"], is_in_both=False)
+    add_ips = create_ip_list(
+        loop_list=prefix["datasource_ips"], check_list=prefix["all_ips"], is_in_both=False
+    )
 
     # If ip-addresses have been found
     if add_ips:
         # Create the payload to create the ip-addresses
-        payload = create_nb_ip_payload(parent_prefix=prefix, data=add_ips, desired_status="auto_discovered")
+        payload = create_nb_ip_payload(
+            parent_prefix=prefix, data=add_ips, ds=ds, desired_status="auto_discovered"
+        )
         # POST request to update the ip-addresses
         resp = post_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", payload=payload)
 
@@ -409,7 +493,7 @@ def update_discovered_ip_addresses(nb_url: str, prefix: dict) -> tuple:
         text = "The following 'Auto-Discovered' ip-addresses had been added:"
         # The function returns the result list and True if the response is successful else False
         sub_result, sub_failed = create_nb_response_result(
-            resp=resp, nb_type="ip", data=add_ips, task_text=task_text, text=text
+            resp=resp, nb_type="ip", data=add_ips, ds=ds, task_text=task_text, text=text
         )
         if "Response Json:" in sub_result[0]:
             sub_result[0] += f"\n** DEBUG **\nPayload:\n{payload}\nResponse:\n{resp.json()}\n** DEBUG **\n"
@@ -418,14 +502,14 @@ def update_discovered_ip_addresses(nb_url: str, prefix: dict) -> tuple:
 
     # Update the ip-addresses with the status 'auto_discovered' that are part of the nmap scan list
     update_ips = create_ip_list(
-        loop_list=prefix["discovered_ips"], check_list=prefix["nmap_ips"], is_in_both=True
+        loop_list=prefix["discovered_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
 
     # If ip-addresses have been found
     if update_ips:
         # Create the payload to create the ip-addresses
         payload = create_nb_ip_payload(
-            parent_prefix=prefix, data=update_ips, desired_status="auto_discovered"
+            parent_prefix=prefix, data=update_ips, ds=ds, desired_status="auto_discovered"
         )
         # PATCH request to update the ip-addresses
         resp = patch_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", payload=payload)
@@ -434,7 +518,7 @@ def update_discovered_ip_addresses(nb_url: str, prefix: dict) -> tuple:
         text = "The following 'Auto-Discovered' ip-addresses had been updated:"
         # The function returns the result list and True if the response is successful else False
         sub_result, sub_failed = create_nb_response_result(
-            resp=resp, nb_type="ip", data=update_ips, task_text=task_text, text=text
+            resp=resp, nb_type="ip", data=update_ips, ds=ds, task_text=task_text, text=text
         )
         result.extend(sub_result)
         failed = True if sub_failed else failed
@@ -442,7 +526,9 @@ def update_discovered_ip_addresses(nb_url: str, prefix: dict) -> tuple:
     return result, failed
 
 
-def delete_inactive_auto_discovered_ip_addresses(nb_url: str, prefix: dict) -> tuple:
+def delete_inactive_auto_discovered_ip_addresses(
+    nb_url: str, prefix: dict, ds: Literal["nmap", "ip-fabric"]
+) -> tuple:
     """
     Deletes inactive auto-discovered IP addresses from NetBox.
 
@@ -460,7 +546,7 @@ def delete_inactive_auto_discovered_ip_addresses(nb_url: str, prefix: dict) -> t
 
     # Delete the ip-addresses with the status 'auto_discovered' that are not in the nmap scan list
     delete_ips = create_ip_list(
-        loop_list=prefix["discovered_ips"], check_list=prefix["nmap_ips"], is_in_both=False
+        loop_list=prefix["discovered_ips"], check_list=prefix["datasource_ips"], is_in_both=False
     )
 
     # If ip-addresses have been found
@@ -473,13 +559,13 @@ def delete_inactive_auto_discovered_ip_addresses(nb_url: str, prefix: dict) -> t
         text = "The following 'Auto-Discovered' ip-addresses had been deleted:"
         # The function returns the result list and True if the response is successful else False
         result, failed = create_nb_response_result(
-            resp=resp, nb_type="ip", data=delete_ips, task_text=task_text, text=text
+            resp=resp, nb_type="ip", data=delete_ips, ds=ds, task_text=task_text, text=text
         )
 
     return result, failed
 
 
-def update_reserved_ip_addresses(nb_url: str, prefix: dict) -> tuple:
+def update_reserved_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap", "ip-fabric"]) -> tuple:
     """
     Updates the status of reserved IP addresses in NetBox if they are reachable by nmap.
 
@@ -497,13 +583,13 @@ def update_reserved_ip_addresses(nb_url: str, prefix: dict) -> tuple:
 
     # Update the ip-addresses with the status 'reserved' that are part of the nmap scan list
     update_ips = create_ip_list(
-        loop_list=prefix["reserved_ips"], check_list=prefix["nmap_ips"], is_in_both=True
+        loop_list=prefix["reserved_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
 
     # If ip-addresses have been found
     if update_ips:
         # Create the payload to update the ip-addresses
-        payload = create_nb_ip_payload(parent_prefix=prefix, data=update_ips, desired_status="active")
+        payload = create_nb_ip_payload(parent_prefix=prefix, data=update_ips, ds=ds, desired_status="active")
         # PATCH request to update the ip-addresses
         resp = patch_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", payload=payload)
 
@@ -511,13 +597,13 @@ def update_reserved_ip_addresses(nb_url: str, prefix: dict) -> tuple:
         text = "The following 'Reserved' ip-addresses had been set to status 'Active':"
         # The function returns the result list and True if the response is successful else False
         result, failed = create_nb_response_result(
-            resp=resp, nb_type="ip", data=update_ips, task_text=task_text, text=text
+            resp=resp, nb_type="ip", data=update_ips, ds=ds, task_text=task_text, text=text
         )
 
     return result, failed
 
 
-def update_inactive_ip_addresses(nb_url: str, prefix: dict) -> tuple:
+def update_inactive_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap", "ip-fabric"]) -> tuple:
     """
     Updates the status of inactive IP addresses in NetBox if they are reachable by nmap.
 
@@ -535,13 +621,15 @@ def update_inactive_ip_addresses(nb_url: str, prefix: dict) -> tuple:
 
     # Update the ip-addresses with the status 'inactive' that are part of the nmap scan list
     inactive_ips = create_ip_list(
-        loop_list=prefix["inactive_ips"], check_list=prefix["nmap_ips"], is_in_both=True
+        loop_list=prefix["inactive_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
 
     # If ip-addresses have been found
     if inactive_ips:
         # Create the payload to update the ip-addresses
-        payload = create_nb_ip_payload(parent_prefix=prefix, data=inactive_ips, desired_status="active")
+        payload = create_nb_ip_payload(
+            parent_prefix=prefix, data=inactive_ips, ds=ds, desired_status="active"
+        )
         # PATCH request to update the ip-addresses
         resp = patch_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", payload=payload)
 
@@ -549,13 +637,15 @@ def update_inactive_ip_addresses(nb_url: str, prefix: dict) -> tuple:
         text = "The following 'Inactive' ip-addresses had been set to status 'Active':"
         # The function returns the result list and True if the response is successful else False
         result, failed = create_nb_response_result(
-            resp=resp, nb_type="ip", data=inactive_ips, task_text=task_text, text=text
+            resp=resp, nb_type="ip", data=inactive_ips, ds=ds, task_text=task_text, text=text
         )
 
     return result, failed
 
 
-def update_active_ip_addresses(nb_url: str, prefix: dict, overwrite_active: list[str]) -> tuple:
+def update_active_ip_addresses(
+    nb_url: str, prefix: dict, overwrite_active: list[str], ds: Literal["nmap", "ip-fabric"]
+) -> tuple:
     """
     Updates the status of active IP addresses in NetBox if they are not reachable by nmap.
 
@@ -573,13 +663,13 @@ def update_active_ip_addresses(nb_url: str, prefix: dict, overwrite_active: list
 
     # Update the ip-addresses with the status 'active' that are part of the nmap scan list
     active_ips = create_ip_list(
-        loop_list=prefix["active_ips"], check_list=prefix["nmap_ips"], is_in_both=True
+        loop_list=prefix["active_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
 
     # If ip-addresses have been found
     if active_ips:
         # Create the payload to update the ip-addresses
-        payload = create_nb_ip_payload(parent_prefix=prefix, data=active_ips, desired_status="active")
+        payload = create_nb_ip_payload(parent_prefix=prefix, data=active_ips, ds=ds, desired_status="active")
         # PATCH request to update the ip-addresses
         resp = patch_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", payload=payload)
 
@@ -587,14 +677,14 @@ def update_active_ip_addresses(nb_url: str, prefix: dict, overwrite_active: list
         text = "The following 'Active' ip-addresses had been updated:"
         # The function returns the result list and True if the response is successful else False
         sub_result, sub_failed = create_nb_response_result(
-            resp=resp, nb_type="ip", data=active_ips, task_text=task_text, text=text
+            resp=resp, nb_type="ip", data=active_ips, ds=ds, task_text=task_text, text=text
         )
         result.extend(sub_result)
         failed = True if sub_failed else failed
 
     # Update the ip-addresses with the status 'active' that are not part of the nmap scan list
     inactive_ips = create_ip_list(
-        loop_list=prefix["active_ips"], check_list=prefix["nmap_ips"], is_in_both=False
+        loop_list=prefix["active_ips"], check_list=prefix["datasource_ips"], is_in_both=False
     )
     # Create a new list to exclude the overwrite_active ip-addresses
     inactive_ips = [ip for ip in inactive_ips if ip["address"] not in overwrite_active]
@@ -602,7 +692,9 @@ def update_active_ip_addresses(nb_url: str, prefix: dict, overwrite_active: list
     # If ip-addresses have been found
     if inactive_ips:
         # Create the payload to update the ip-addresses
-        payload = create_nb_ip_payload(parent_prefix=prefix, data=inactive_ips, desired_status="inactive")
+        payload = create_nb_ip_payload(
+            parent_prefix=prefix, data=inactive_ips, ds=ds, desired_status="inactive"
+        )
         # PATCH request to update the ip-addresses
         resp = patch_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", payload=payload)
 
@@ -610,7 +702,7 @@ def update_active_ip_addresses(nb_url: str, prefix: dict, overwrite_active: list
         text = "The following 'Active' ip-addresses had been set to status 'Inactive':"
         # The function returns the result list and True if the response is successful else False
         sub_result, sub_failed = create_nb_response_result(
-            resp=resp, nb_type="ip", data=inactive_ips, task_text=task_text, text=text
+            resp=resp, nb_type="ip", data=inactive_ips, ds=ds, task_text=task_text, text=text
         )
         result.extend(sub_result)
         failed = True if sub_failed else failed
@@ -639,7 +731,7 @@ def set_results_changed_failed(results, result, changed, sub_failed, failed) ->
     return results, changed, failed
 
 
-def update_active_netbox_prefix_ip_addresses(prefix: list, *overwrite_active) -> tuple:
+def update_netbox_prefix_ip_addresses(prefix: list, *args) -> tuple:
     """
     This function can be used within a ThreadPoolExecutor. Update the IP addresses of a active NetBox prefix
     with the following information:
@@ -656,9 +748,10 @@ def update_active_netbox_prefix_ip_addresses(prefix: list, *overwrite_active) ->
     """
     # Create a list to collect the task results
     results = []
-    # overwrite_active is a tuple as its passed by the thread pool executor via the *args
-    # Therefor the tuple will be changes into a list
-    overwrite_active = list(overwrite_active)
+    # Set the datasource by the first arg ('namp' or 'ip-fabric') and make sure its a string
+    ds = str(args[0])
+    # Set the ip-address list to overwrite the status as active by the second arg and make sure its a list
+    overwrite_active = list(args[1])
     # Boolian to check if any ip-addresses have been changed and the overall failed status
     changed = False
     failed = False
@@ -669,29 +762,31 @@ def update_active_netbox_prefix_ip_addresses(prefix: list, *overwrite_active) ->
     # Get the base url of the NetBox instance
     nb_url = base_url(url=prefix["url"], with_path=False)
 
-    # Get all NetBox ip-addresses, scan the prefix with nmap add lists to the prefix dict
-    result, prefix = get_netbox_ip_addresses_and_scan_prefix(nb_url=nb_url, prefix=prefix)
+    # Scan the prefix with nmap and add the list to the prefix dict or get the ip-addresses from IP-Fabric
+    result, prefix, failed = get_nb_ips_and_external_datasource(nb_url=nb_url, prefix=prefix, ds=ds)
     results.extend(result)
+    if failed:
+        return results, failed
 
     # Add new 'auto-discovered' ip-addresses
-    result, sub_failed = update_discovered_ip_addresses(nb_url=nb_url, prefix=prefix)
+    result, sub_failed = update_discovered_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
     # Delete inactive 'auto-discovered' ip-addresses
-    result, sub_failed = delete_inactive_auto_discovered_ip_addresses(nb_url=nb_url, prefix=prefix)
+    result, sub_failed = delete_inactive_auto_discovered_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
     # Update 'reserved' ip-addresses -> set status to 'active'
-    result, sub_failed = update_reserved_ip_addresses(nb_url=nb_url, prefix=prefix)
+    result, sub_failed = update_reserved_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
     # Update 'inactive' ip-addresses -> set status to 'active'
-    result, sub_failed = update_inactive_ip_addresses(nb_url=nb_url, prefix=prefix)
+    result, sub_failed = update_inactive_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
     # Update 'active' ip-addresses -> set status to 'active' or 'inactive'
     result, sub_failed = update_active_ip_addresses(
-        nb_url=nb_url, prefix=prefix, overwrite_active=overwrite_active
+        nb_url=nb_url, prefix=prefix, overwrite_active=overwrite_active, ds=ds
     )
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
@@ -816,7 +911,9 @@ def update_all_netbox_vlans(vlan: dict) -> tuple:
     return results, failed
 
 
-def run_thread_pool(title: str, task: Callable, thread_list: list[dict], args: tuple = ()) -> list:
+def run_thread_pool(
+    title: str, task: Callable, thread_list: list[dict], max_workers: int = 50, args: tuple = ()
+) -> list:
     """
     Runs a thread pool with a given task for each item in the thread list.
 
@@ -843,7 +940,7 @@ def run_thread_pool(title: str, task: Callable, thread_list: list[dict], args: t
         return []
 
     # Create a ThreadPoolExecutor
-    with ThreadPoolExecutor(max_workers=100) as executor:
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
         # Submit a new task for each prefix to update and collect the tasks results
         threads = [executor.submit(task, item, *args) for item in thread_list]
 
@@ -881,7 +978,7 @@ def print_thread_pool_results(title: str, thread_result, fail_hard: bool = False
     return failed_task
 
 
-def main(nr_config: str, overwrite_active: list[str] = None) -> None:
+def main(nr_config: str, nmap_scan: bool = False, overwrite_active: list[str] = None) -> None:
     """
     Main function is intended to import and execute by other scripts.
     It loads NetBox inventory, scan active prefixes, and update IP addresses and VLANs.
@@ -918,11 +1015,20 @@ def main(nr_config: str, overwrite_active: list[str] = None) -> None:
     # Get the NetBox URL (Authentication token will be loaded as nb_token env variable)
     nb_url = nr_config_dict["inventory"]["options"]["nb_url"]
 
-    # Load Active NetBox Prefixes
-    nb_active_prefixes = load_netbox_data(
-        task_text="Load Active NetBox Prefixes",
+    # Load Active NetBox Prefixes from Tenant 'none', 'oob', 'tier-1' and 'tier-2'
+    # These prefixes will be scanned with nmap and updated
+    nb_active_oob_t1_t2_prefixes = load_netbox_data(
+        task_text="Load Active OOB/T1/T2 NetBox Prefixes",
+        nb_api_url=f"{nb_url}/api/ipam/prefixes/",
+        query={"status": "active", "tenant": ["none", "oob", "tier-1", "tier-2"], "mark_utilized": "false"},
+    )
+
+    # Load Active/Inventory NetBox Prefixes from Tenant 'tier-3' and 'tier-4'
+    # These prefixes will be updated with input from IP-Fabric
+    nb_active_inventory_t3_t4_prefixes = load_netbox_data(
+        task_text="Load Active & Inventory T3/T4 NetBox Prefixes",
         nb_api_url=f"{nb_url}/api/ipam/prefixes/",
-        query={"status": "active", "mark_utilized": "false"},
+        query={"status": ["active", "inventory"], "tenant": ["tier-3", "tier-4"], "mark_utilized": "false"},
     )
 
     # Load NetBox Non-Container Prefixes
@@ -940,20 +1046,42 @@ def main(nr_config: str, overwrite_active: list[str] = None) -> None:
         query={},
     )
 
-    #### Scan Active NetBox Prefixes and Update IP-Addresses Status, DNS-Name and Ports ######################
+    #### Nmap Scan Active NetBox Prefixes and Update IP-Addresses ###########################################
+
+    if nmap_scan:
+        # Set the task title
+        title = "Nmap Scan Active OOB/T1/T2 NetBox Prefixes and Update IP-Addresses"
+
+        # Run the thread pool to update all NetBox IP-Addresses Status and DNS-Name
+        # 1. arg is the input type ('nmap' or 'ip-fabric')
+        # 2. arg is the list of active IP addresses to overwrite
+        thread_result = run_thread_pool(
+            title=title,
+            task=update_netbox_prefix_ip_addresses,
+            thread_list=nb_active_oob_t1_t2_prefixes,
+            max_workers=50,
+            args=("nmap", overwrite_active),
+        )
+        # Print the thread pool results and exit the script if any task has failed
+        print_thread_pool_results(title=title, thread_result=thread_result, fail_hard=True)
+
+    #### IP-Fabric Update Active & Inventory NetBox Prefixes IP-Addresses ###################################
 
     # Set the task title
-    title = "Scan Active NetBox Prefixes and Update IP-Addresses Status, DNS-Name and Ports"
+    title = "IP-Fabric Update Active & Inventory T3/T4 NetBox Prefixes IP-Addresses"
 
     # Run the thread pool to update all NetBox IP-Addresses Status and DNS-Name
+    # 1. arg is the input type ('nmap' or 'ip-fabric')
+    # 2. arg is the list of active IP addresses to overwrite
     thread_result = run_thread_pool(
         title=title,
-        task=update_active_netbox_prefix_ip_addresses,
-        thread_list=nb_active_prefixes,
-        args=overwrite_active,
+        task=update_netbox_prefix_ip_addresses,
+        thread_list=nb_active_inventory_t3_t4_prefixes,
+        max_workers=10,
+        args=("ip-fabric", overwrite_active),
     )
     # Print the thread pool results and exit the script if any task has failed
-    print_thread_pool_results(title=title, thread_result=thread_result, fail_hard="hard")
+    print_thread_pool_results(title=title, thread_result=thread_result, fail_hard=True)
 
     #### Update all IP-Addresses Tags and other Fields ######################################################
 
@@ -964,10 +1092,11 @@ def main(nr_config: str, overwrite_active: list[str] = None) -> None:
     thread_result = run_thread_pool(
         title=title,
         task=update_all_netbox_ip_addresses,
+        max_workers=100,
         thread_list=nb_subnet_prefixes,
     )
     # Print the thread pool results and exit the script if any task has failed
-    print_thread_pool_results(title=title, thread_result=thread_result, fail_hard="hard")
+    print_thread_pool_results(title=title, thread_result=thread_result, fail_hard=True)
 
     #### Update all VLANs Tags and other Fields #############################################################
 
@@ -978,7 +1107,8 @@ def main(nr_config: str, overwrite_active: list[str] = None) -> None:
     thread_result = run_thread_pool(
         title=title,
         task=update_all_netbox_vlans,
+        max_workers=100,
         thread_list=nb_vlans,
     )
     # Print the thread pool results and exit the script if any task has failed
-    print_thread_pool_results(title=title, thread_result=thread_result, fail_hard="hard")
+    print_thread_pool_results(title=title, thread_result=thread_result, fail_hard=True)

{nornir_collection-0.0.10 → nornir_collection-0.0.12}/nornir_collection/netbox/utils.py

@@ -104,6 +104,64 @@ def init_args_for_nornir_config_filepath() -> str:
     return nr_config
 
 
+def init_args_for_ipam_update() -> str:
+    """
+    This function initialze arguments to specify which NetBox instance and Nornir config filepath to use.
+    """
+    task_text = "Argparse verify arguments"
+    print_task_name(text=task_text)
+
+    # Load environment variables or raise a TypeError when is None
+    env_vars = get_env_vars(envs=["NR_CONFIG_PROD", "NR_CONFIG_TEST"], task_text=task_text)
+    nr_config_prod = env_vars["NR_CONFIG_PROD"]
+    nr_config_test = env_vars["NR_CONFIG_TEST"]
+
+    # Define the arguments which needs to be given to the script execution
+    argparser = CustomArgParse(
+        prog=os.path.basename(__main__.__file__),
+        description="Specify the NetBox PROD or TEST instance and Nornir config filepath to be used",
+        epilog="One of the two mandatory arguments is required.",
+        argument_default=argparse.SUPPRESS,
+        formatter_class=CustomArgParseWidthFormatter,
+    )
+    # Add all NetBox arguments
+    argparser.add_argument(
+        "--prod",
+        action="store_true",
+        help=f"use the NetBox 'PROD' instance and Nornir config '{nr_config_prod}'",
+    )
+    argparser.add_argument(
+        "--test",
+        action="store_true",
+        help=f"use the NetBox 'TEST' instance and Nornir config '{nr_config_test}'",
+    )
+    # Add the optional rebuild argument
+    argparser.add_argument(
+        "--nmap",
+        action="store_true",
+        default=False,
+        help="enable additionally to IP-Fabric a NMAP scan (default: no NMAP scan)",
+    )
+    # Verify the provided arguments and print the custom argparse error message in case any error or wrong
+    # arguments are present and exit the script
+    args = argparser.parse_args()
+
+    # Verify the NetBox instance and Nornir config filepath
+    if not (hasattr(args, "prod") or hasattr(args, "test")):
+        argparser.error("No NetBox instance specified, add --prod or --test")
+
+    # Set the NetBox instance and the Nornir config file based on the arguments
+    nb_instance = "TEST" if args.test else "PROD"
+    nr_config = nr_config_test if args.test else nr_config_prod
+
+    # If argparser.parse_args() is successful -> no argparse error message
+    print(task_info(text=task_text, changed=False))
+    print(f"'{task_text}' -> ArgparseResponse <Success: True>")
+    print(f"-> Run on the NetBox '{nb_instance}' instance and Nornir config '{nr_config}'")
+
+    return nr_config, args
+
+
 def get_nb_resources(url: str, params: Dict[str, Any] = {}) -> List[Dict[str, Any]]:
     """
     TBD

{nornir_collection-0.0.10 → nornir_collection-0.0.12}/nornir_collection.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nornir-collection
-Version: 0.0.10
+Version: 0.0.12
 Summary: Nornir-Collection contains network automation functions and complete IaC workflows with         Nornir and other python libraries. It contains Nornir tasks and general functions in Nornir style.
 Author: Willi Kubny
 Author-email: willi.kubny@gmail.ch
@@ -12,7 +12,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: nornir==3.3.0
 Requires-Dist: nornir-jinja2==0.2.0
-Requires-Dist: nornir-salt==0.20.3
+Requires-Dist: nornir-salt==0.22.2
 Requires-Dist: nornir-netmiko==1.0.1
 Requires-Dist: nornir-scrapli==2023.7.30
 Requires-Dist: scrapli-netconf==2023.7.30
@@ -20,6 +20,7 @@ Requires-Dist: scrapli[ssh2]
 Requires-Dist: scrapli[genie]
 Requires-Dist: nornir-utils==0.2.0
 Requires-Dist: pynetbox==7.4.0
+Requires-Dist: ipfabric==7.0.2
 Requires-Dist: py-pure-client==1.51.0
 Requires-Dist: nornir-pyfgt==1.0.3
 Requires-Dist: python-nmap==0.7.1

{nornir_collection-0.0.10 → nornir_collection-0.0.12}/nornir_collection.egg-info/SOURCES.txt

@@ -39,12 +39,12 @@ setup.py
 ./nornir_collection/netbox/__init__.py
 ./nornir_collection/netbox/custom_script.py
 ./nornir_collection/netbox/inventory.py
-./nornir_collection/netbox/scan_prefixes_and_update_ip_addresses.py
 ./nornir_collection/netbox/set_device_status.py
 ./nornir_collection/netbox/sync_datasource.py
 ./nornir_collection/netbox/update_cisco_inventory_data.py
 ./nornir_collection/netbox/update_cisco_support_plugin_data.py
 ./nornir_collection/netbox/update_fortinet_inventory_data.py
+./nornir_collection/netbox/update_prefixes_ip_addresses.py
 ./nornir_collection/netbox/update_purestorage_inventory_data.py
 ./nornir_collection/netbox/utils.py
 ./nornir_collection/netbox/verify_device_primary_ip.py
@@ -98,12 +98,12 @@ nornir_collection/fortinet/utils.py
 nornir_collection/netbox/__init__.py
 nornir_collection/netbox/custom_script.py
 nornir_collection/netbox/inventory.py
-nornir_collection/netbox/scan_prefixes_and_update_ip_addresses.py
 nornir_collection/netbox/set_device_status.py
 nornir_collection/netbox/sync_datasource.py
 nornir_collection/netbox/update_cisco_inventory_data.py
 nornir_collection/netbox/update_cisco_support_plugin_data.py
 nornir_collection/netbox/update_fortinet_inventory_data.py
+nornir_collection/netbox/update_prefixes_ip_addresses.py
 nornir_collection/netbox/update_purestorage_inventory_data.py
 nornir_collection/netbox/utils.py
 nornir_collection/netbox/verify_device_primary_ip.py

{nornir_collection-0.0.10 → nornir_collection-0.0.12}/nornir_collection.egg-info/requires.txt

@@ -1,6 +1,6 @@
 nornir==3.3.0
 nornir-jinja2==0.2.0
-nornir-salt==0.20.3
+nornir-salt==0.22.2
 nornir-netmiko==1.0.1
 nornir-scrapli==2023.7.30
 scrapli-netconf==2023.7.30
@@ -8,6 +8,7 @@ scrapli[ssh2]
 scrapli[genie]
 nornir-utils==0.2.0
 pynetbox==7.4.0
+ipfabric==7.0.2
 py-pure-client==1.51.0
 nornir-pyfgt==1.0.3
 python-nmap==0.7.1

{nornir_collection-0.0.10 → nornir_collection-0.0.12}/setup.py

@@ -11,7 +11,7 @@ with open("requirements.txt", encoding="utf-8") as f:
 
 setuptools.setup(
     name="nornir-collection",
-    version="0.0.10",
+    version="0.0.12",
     author="Willi Kubny",
     author_email="willi.kubny@gmail.ch",
     description="Nornir-Collection contains network automation functions and complete IaC workflows with \