nornir-collection 0.0.27__tar.gz → 0.0.29__tar.gz

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nornir-collection
-Version: 0.0.27
+Version: 0.0.29
 Summary: Nornir-Collection contains network automation functions and complete IaC workflows with         Nornir and other python libraries. It contains Nornir tasks and general functions in Nornir style.
 Author: Willi Kubny
 Author-email: willi.kubny@gmail.ch
@@ -10,19 +10,19 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: nornir==3.3.0
+Requires-Dist: nornir==3.5.0
 Requires-Dist: nornir-jinja2==0.2.0
-Requires-Dist: nornir-salt==0.22.2
+Requires-Dist: nornir-salt==0.22.5
 Requires-Dist: nornir-netmiko==1.0.1
-Requires-Dist: nornir-scrapli==2023.7.30
-Requires-Dist: scrapli-netconf==2023.7.30
+Requires-Dist: nornir-scrapli==2024.7.30
+Requires-Dist: scrapli-netconf==2024.1.30
+Requires-Dist: nornir-utils==0.2.0
+Requires-Dist: nornir-pyfgt==1.0.3
 Requires-Dist: scrapli[ssh2]
 Requires-Dist: scrapli[genie]
-Requires-Dist: nornir-utils==0.2.0
 Requires-Dist: pynetbox==7.4.0
-Requires-Dist: ipfabric==7.0.2
+Requires-Dist: ipfabric==7.2.2
 Requires-Dist: py-pure-client==1.51.0
-Requires-Dist: nornir-pyfgt==1.0.3
 Requires-Dist: python-nmap==0.7.1
 Requires-Dist: pybatfish==2023.12.16.1270
 Requires-Dist: requests==2.31.0
@@ -30,7 +30,7 @@ Requires-Dist: pre-commit==3.6.0
 Requires-Dist: pyyaml==6.0.1
 Requires-Dist: pyfiglet==1.0.2
 Requires-Dist: GitPython==3.1.40
-Requires-Dist: python-dotenv==1.0.0
+Requires-Dist: python-dotenv==1.1.1
 Requires-Dist: beautifultable==1.1.0
 Requires-Dist: pandas==2.2.2
 Requires-Dist: openpyxl==3.1.2

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/nornir_collection/netbox/inventory.py

@@ -248,7 +248,7 @@ def _load_cisco_support_data(task: Task, all_csupp: List[Dict]) -> Result:
 #### Nornir Tasks in regular Function #######################################################################
 
 
-def load_additional_netbox_data(nr: Nornir, add_netbox_data: dict[str:bool]) -> Nornir:
+def load_additional_netbox_data(nr: Nornir, add_netbox_data: dict[str:bool], silent: bool = False) -> Nornir:
     """
     Load additional data from NetBox into Nornir inventory based on 'add_netbox_data' options.
     """
@@ -352,10 +352,11 @@ def load_additional_netbox_data(nr: Nornir, add_netbox_data: dict[str:bool]) ->
         # Append the result message
         result_msg.append("  - Load Interfaces")
 
-    # Print the result of the additional loaded data
-    print_task_name(text=task_text)
-    print(task_info(text=task_text, changed=False))
-    print("'Load NetBox additional inventory data' -> NornirResponse <Success: True>")
-    print("-> Additional loaded data:")
-    for msg in result_msg:
-        print(msg)
+    if not silent:
+        # Print the result of the additional loaded data
+        print_task_name(text=task_text)
+        print(task_info(text=task_text, changed=False))
+        print("'Load NetBox additional inventory data' -> NornirResponse <Success: True>")
+        print("-> Additional loaded data:")
+        for msg in result_msg:
+            print(msg)

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/nornir_collection/netbox/update_prefixes_ip_addresses.py

@@ -6,7 +6,6 @@ The Main function is intended to import and execute by other scripts.
 """
 
 import os
-import sys
 import ipaddress
 import urllib.parse
 from typing import Callable, Literal, Union
@@ -14,19 +13,17 @@ from concurrent.futures import ThreadPoolExecutor
 import requests
 import nmap
 from ipfabric import IPFClient
+from nornir_salt.plugins.processors import TestsProcessor
+from nornir_salt.plugins.functions import FFun
+from nornir_salt.plugins.tasks import scrapli_send_commands
+from nornir_collection.nornir_plugins.inventory.utils import init_nornir
 from nornir_collection.netbox.utils import (
     get_nb_resources,
     post_nb_resources,
     patch_nb_resources,
     delete_nb_resources,
 )
-from nornir_collection.utils import (
-    print_task_title,
-    task_name,
-    exit_error,
-    load_yaml_file,
-    task_result,
-)
+from nornir_collection.utils import print_task_title, task_name, exit_error, task_result
 
 
 __author__ = "Willi Kubny"
@@ -37,6 +34,36 @@ __email__ = "willi.kubny@dreyfusbank.ch"
 __status__ = "Production"
 
 
+def nr_ping_test(nr, hosts: list[str], ip_list: list[str], criteria: Literal["PASS", "FAIL"]) -> list[str]:
+    """
+    TBD
+    """
+    # Create the test suite list for each ip-address to ping
+    tests_suite = [
+        {
+            "name": ip["address"],
+            "task": f"ping {ipaddress.ip_interface(ip['address']).ip}",
+            "test": "!contains_lines",
+            "pattern": ["Success rate is 0", "VRF Mgmt-vrf does not have a usable source address"],
+        }
+        for ip in ip_list
+    ]
+    # Use NornirSalt FFun Filter-List option to filter on a list of hosts
+    nr_filtered = FFun(nr, FL=hosts)
+    # Add the nornir salt TestsProcessor processor
+    nr_with_testsprocessor = nr_filtered.with_processors(
+        [TestsProcessor(tests=tests_suite, build_per_host_tests=True)]
+    )
+    # Run the TestsProcessor task
+    results = nr_with_testsprocessor.run(task=scrapli_send_commands, on_failed=True)
+    # Create a list of the ip-addresses based on the required criteria
+    result_list = [result.name for host in results for result in results[host] if result.result == criteria]
+    # Create a list of ip-addresses that are part of the result_list -> This matchs the criteria PASS/FAIL
+    verified_ip_list = [ip for ip in ip_list if ip["address"] in result_list]
+
+    return verified_ip_list
+
+
 def load_netbox_data(task_text: str, nb_api_url: str, query: dict) -> list[dict]:
     """
     Load NetBox data using the provided task text, NetBox API URL, and query parameters.
@@ -246,9 +273,9 @@ def create_nb_ip_payload(
                     md.extend(
                         [[f"{k}/tcp", v["state"], v["name"]] for k, v in datasource_ip[0]["ports"].items()]
                     )
-                    item["custom_fields"] = {"ipaddress_ports": make_markdown_table(md)}
+                    item["custom_fields"] = {"ipam_ports": make_markdown_table(md)}
                 else:
-                    item["custom_fields"] = {"ipaddress_ports": None}
+                    item["custom_fields"] = {"ipam_ports": None}
         # If the desired_status is None, the payload is for all ip addresses of a prefix
         else:
             # Add the 'vrf' to the payload
@@ -306,25 +333,28 @@ def nmap_scan_host_ip_or_subnet(hosts: str) -> list:
     return nmap_scan_result
 
 
-def nmap_double_check_ips(ip_list: dict) -> list:
+def nmap_double_check_ips(ip_list: dict, criteria: Literal["PASS", "FAIL"]) -> list:
     """
     TBD
     """
     verified_ips = []
-    # Nmap scan ip-addresses of the maybe_delete_ips list
+    # Nmap scan ip-addresses of the ip list
     for ip in ip_list:
         # Scan the prefix with nmap
         scan_result = nmap_scan_host_ip_or_subnet(hosts=ip["address"])
         # Add the nmap scan result to the inactive_ips list with the ID of the ip-address
-        if scan_result:
+        if scan_result and criteria == "PASS":
             # As a single ip-address is scanned the scan_result can have only one list item
             verified_ips.append({"id": ip["id"], **scan_result[0]})
+        elif not scan_result and criteria == "FAIL":
+            # As the single ip-address is not reachable, add the ip-address to the verified_ips list
+            verified_ips.append(ip)
 
-    # Return the verified ip-addresses list
+    # Return the active ip-addresses list
     return verified_ips
 
 
-def get_ipfabric_data_for_prefix(prefix: dict) -> tuple:
+def get_ipfabric_ip_addresses_for_prefix(prefix: dict) -> tuple:
     """
     TBD
     """
@@ -402,7 +432,7 @@ def get_nb_ips_and_external_datasource(nb_url: str, prefix: dict, ds: Literal["n
         )
     elif ds == "ip-fabric":
         # Get the ip-addresses from the IP-Fabric
-        prefix["datasource_ips"] = get_ipfabric_data_for_prefix(prefix=prefix)
+        prefix["datasource_ips"] = get_ipfabric_ip_addresses_for_prefix(prefix=prefix)
 
         # Print the task result
         text = f"IP-Fabric Get Data for Prefix {prefix['prefix']} IP-Addresses"
@@ -644,7 +674,7 @@ def update_reserved_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap",
     return result, failed
 
 
-def update_inactive_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap", "ip-fabric"]) -> tuple:
+def update_inactive_ip_addresses(nr, nb_url: str, prefix: dict, ds: Literal["nmap", "ip-fabric"]) -> tuple:
     """
     Updates the status of inactive IP addresses in NetBox if they are reachable by nmap.
 
@@ -665,12 +695,18 @@ def update_inactive_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap",
         loop_list=prefix["inactive_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
     # Double-check ip-addresses with the status 'inactive' that are not part of the datasource list
-    maybe_active_ips = create_ip_list(
+    may_active_ips = create_ip_list(
         loop_list=prefix["inactive_ips"], check_list=prefix["datasource_ips"], is_in_both=False
     )
-    # Nmap scan ip-addresses of the maybe_active_ips list
-    nmap_result = nmap_double_check_ips(ip_list=maybe_active_ips)
-    active_ips.extend(nmap_result)
+    if may_active_ips and ds == "ip-fabric":
+        # Double-check ip-addresses which can be scanned with nmap (all except Tier-3 and Tier-4)
+        if ds == "ip-fabric" and (prefix["tenant"]["slug"] not in ("tier-3", "tier-4")):
+            # Nmap scan ip-addresses of the may_active_ips list and add the result to the active_ips list
+            active_ips.extend(nmap_double_check_ips(ip_list=may_active_ips, criteria="PASS"))
+        # Double-check ip-addresses with a Ping from the core-switch (Tier-3 and Tier-4)
+        elif ds == "ip-fabric" and (prefix["tenant"]["slug"] in ("tier-3", "tier-4")):
+            t3_core_sw = ["T3-AES99-SWRSU1"]
+            active_ips.extend(nr_ping_test(nr=nr, hosts=t3_core_sw, ip_list=may_active_ips, criteria="PASS"))
 
     # If ip-addresses have been found
     if active_ips:
@@ -690,7 +726,7 @@ def update_inactive_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap",
 
 
 def update_active_ip_addresses(
-    nb_url: str, prefix: dict, overwrite_active: list[str], ds: Literal["nmap", "ip-fabric"]
+    nr, nb_url: str, prefix: dict, overwrite_active: list[str], ds: Literal["nmap", "ip-fabric"]
 ) -> tuple:
     """
     Updates the status of active IP addresses in NetBox if they are not reachable by nmap.
@@ -728,16 +764,25 @@ def update_active_ip_addresses(
         result.extend(sub_result)
         if sub_failed:
             failed = True
+            result.append(f"-> Data for Payload:\n{active_ips}")
             result.append(f"-> Payload:\n{payload}")
 
     # Update the ip-addresses with the status 'active' that are not part of the datacenter list
-    maybe_inactive_ips = create_ip_list(
+    may_inactive = create_ip_list(
         loop_list=prefix["active_ips"], check_list=prefix["datasource_ips"], is_in_both=False
     )
-    # Nmap scan ip-addresses of the maybe_inactive_ips list
-    nmap_result = nmap_double_check_ips(ip_list=maybe_inactive_ips)
+    if may_inactive and ds == "ip-fabric":
+        # Double-check ip-addresses which can be scanned with nmap (all except Tier-3 and Tier-4)
+        if prefix["tenant"]["slug"] not in ("tier-3", "tier-4"):
+            # Nmap scan ip-addresses of the may_inactive list
+            may_inactive = nmap_double_check_ips(ip_list=may_inactive, criteria="FAIL")
+        # Double-check ip-addresses with a Ping from the core-switch (Tier-3 and Tier-4)
+        elif prefix["tenant"]["slug"] in ("tier-3", "tier-4"):
+            t3_core_sw = ["T3-AES99-SWRSU1"]
+            may_inactive = nr_ping_test(nr=nr, hosts=t3_core_sw, ip_list=may_inactive, criteria="FAIL")
+
     # Create a new list to exclude the overwrite_active ip-addresses
-    inactive_ips = [ip for ip in nmap_result if ip["address"] not in overwrite_active]
+    inactive_ips = [ip for ip in may_inactive if ip["address"] not in overwrite_active]
 
     # If ip-addresses have been found
     if inactive_ips:
@@ -805,6 +850,8 @@ def update_netbox_prefix_ip_addresses(prefix: list, *args) -> tuple:
     ds = str(args[0])
     # Set the ip-address list to overwrite the status as active by the second arg and make sure its a list
     overwrite_active = list(args[1])
+    # Set the Nornir config file by the third arg
+    nr_config = args[2]
     # Boolian to check if any ip-addresses have been changed and the overall failed status
     changed = False
     failed = False
@@ -812,8 +859,12 @@ def update_netbox_prefix_ip_addresses(prefix: list, *args) -> tuple:
     # Print the task title
     results.append(task_name(text=f"Update NetBox IP-Addresses of Prefix {prefix['prefix']}"))
 
-    # Get the base url of the NetBox instance
-    nb_url = base_url(url=prefix["url"], with_path=False)
+    # Initialize, transform and filter the Nornir inventory are return the filtered Nornir object
+    # Define data to load from NetBox in addition to the base Nornir inventory plugin
+    add_netbox_data = {"load_virtual_chassis_data": True}
+    nr = init_nornir(config_file=nr_config, add_netbox_data=add_netbox_data, silent=True)
+    # Get the NetBox url from the inventory options
+    nb_url = nr.config.inventory.options["nb_url"]
 
     # Scan the prefix with nmap and add the list to the prefix dict or get the ip-addresses from IP-Fabric
     result, prefix, failed = get_nb_ips_and_external_datasource(nb_url=nb_url, prefix=prefix, ds=ds)
@@ -833,13 +884,13 @@ def update_netbox_prefix_ip_addresses(prefix: list, *args) -> tuple:
     result, sub_failed = update_reserved_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
-    # Update 'inactive' ip-addresses -> set status to 'active' (double check with partial nmap scan)
-    result, sub_failed = update_inactive_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
+    # Update 'inactive' ip-addresses -> set status to 'active' (with partial double-check)
+    result, sub_failed = update_inactive_ip_addresses(nr=nr, nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
-    # Update 'active' ip-addresses -> set status to 'active' or 'inactive' (check with partial nmap scan)
+    # Update 'active' ip-addresses -> set status to 'active' or 'inactive' (with partial double-check)
     result, sub_failed = update_active_ip_addresses(
-        nb_url=nb_url, prefix=prefix, overwrite_active=overwrite_active, ds=ds
+        nr=nr, nb_url=nb_url, prefix=prefix, overwrite_active=overwrite_active, ds=ds
     )
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
@@ -855,6 +906,139 @@ def update_netbox_prefix_ip_addresses(prefix: list, *args) -> tuple:
     return results, failed
 
 
+def update_netbox_ip_address_switchport(prefix: list, *args) -> tuple:
+    """
+    This function can be used within a ThreadPoolExecutor. Update additional the IP addresses of a active
+    NetBox Prefix with the following information about where this IP-Address is activ in the network:
+    - Switch
+    - Switchport
+    - Switchport Config-Mode
+
+    Args:
+        prefix (dict): The prefix to update, containing information about the prefix.
+        nb_url (str): The URL of the NetBox instance.
+
+    Returns:
+        list: A tuple of results from the update tasks and a boolean indicating if the task failed.
+    """
+    # Create a list to collect the task results
+    results = []
+    # Boolian to check if any ip-addresses have been changed and the overall failed status
+    failed = False
+    # Set the IP-Fabric MAC addresses by the first arg
+    ipf_edge_mac_dict = args[0]
+    # Set the Nornir object by the second arg
+    nr = args[1]
+
+    # Print the task title
+    results.append(task_name(text=f"Update NetBox IP-Addresses of Prefix {prefix['prefix']}"))
+
+    # Get the base url of the NetBox instance
+    nb_url = base_url(url=prefix["url"], with_path=False)
+
+    # Add a list of dicts (id & address) of NetBox ip-addresses with the status 'active' for prefix
+    query = {"parent": prefix["prefix"], "status": "active"}
+    resp = get_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", params=query)
+    prefix["active_ips"] = [{"id": i["id"], "address": i["address"]} for i in resp] if resp else []
+
+    # Print the task result
+    text = f"Get NetBox IP-Addresses of Prefix {prefix['prefix']}"
+    if len(prefix["active_ips"]) == 0:
+        results.append(
+            f"{task_result(text=text, changed=False, level_name='INFO')}\n"
+            + f"'{text}' -> NetBoxResponse <Success: True>\n"
+            + f"-> Prefix {prefix['prefix']} have no ip-addresses to update"
+        )
+        # Return the results and status
+        return results, failed
+    results.append(
+        f"{task_result(text=text, changed=False, level_name='INFO')}\n"
+        + f"'{text}' -> NetBoxResponse <Success: True>\n"
+        + f"-> NetBox active ip-address count: {len(prefix['active_ips'])}"
+    )
+
+    # Connect to IP-Fabric
+    ipf = IPFClient(
+        base_url=os.environ["IPF_URL"], auth=os.environ["IPF_TOKEN"], snapshot_id="$last", verify=False
+    )
+
+    # Get the prefix length from the prefix
+    prefixlen = ipaddress.ip_network(prefix["prefix"]).prefixlen
+
+    # Create some variables to store the data later
+    ip_detail_dict = {}
+    updated_fields = []
+    # Get all ip-addresses and mac-addresses of the prefix from the IP-Fabric technology arp table
+    filters = {"ip": ["cidr", prefix["prefix"]], "proxy": ["eq", False]}
+    columns = ["ip", "mac"]
+    ipf_arp_table = ipf.technology.addressing.arp_table.all(filters=filters, columns=columns)
+    # Create a mac-address dict only for the ip-addresses that are in the ipf_arp_table for the prefix
+    ipf_arp_table = [x for x in ipf_arp_table if x["mac"] in ipf_edge_mac_dict.keys()]
+    results.append(f"-> IP-Fabric edge ip-address count: {len(ipf_arp_table)}")
+
+    for x in ipf_arp_table:
+        # Set some variables for easier code reading
+        switch = ipf_edge_mac_dict[x["mac"]]["hostname"]
+        switch_id = nr.inventory.hosts[switch].get("id")
+        switchport = ipf_edge_mac_dict[x["mac"]]["intName"]
+        # Get the switchport information from IPF
+        ipf_switchport = ipf.technology.interfaces.switchport.all(
+            filters={"hostname": ["eq", switch], "intName": ["eq", switchport]}
+        )
+        if ipf_switchport[0]["mode"] == "static access":
+            config_mode = "Access-Port"
+        elif ipf_switchport[0]["mode"] == "trunk":
+            config_mode = "Trunk-Port"
+        else:
+            config_mode = ipf_switchport[0]["mode"]
+        nw_team_needed = True if config_mode == "Access-Port" else False
+        # Create the ip-detail dict with the infos to ceate the payload
+        ip_detail_dict[f"{x['ip']}/{prefixlen}"] = {
+            "ipam_1_switch": switch_id,
+            "ipam_2_switchport": switchport,
+            "ipam_3_switchport_config_mode": config_mode,
+            "ipam_4_mac": x["mac"],
+            "p0_4_nws_team_needed": nw_team_needed,
+        }
+        updated_fields.append(
+            f"  - {x['ip']}/{prefixlen} (Switch ID: {switch_id}, Switchport: {switchport}, "
+            f"Config-Mode: {config_mode}, MAC: {x['mac']}, NWS-Team: {nw_team_needed})"
+        )
+
+    # Create the payload to update the ip-addresses
+    payload = []
+    for ip in prefix["active_ips"]:
+        # Check if the IP address is in the ip_detail_dict
+        if ip["address"] in ip_detail_dict.keys():
+            payload.append({"id": ip["id"], "custom_fields": {**ip_detail_dict[ip["address"]]}})
+
+    # PATCH request to update the ip-addresses
+    resp = patch_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", payload=payload)
+
+    # Verify the response code and print the result
+    if resp.status_code in [200, 201, 204]:
+        task_text = "Update Edge IP-Addresses Fields"
+        text = "The following edge ip-addresses fields had been updated:"
+        updated_fields = "\n".join(updated_fields)
+        results.append(
+            f"{task_result(text=task_text, changed=False, level_name='INFO')}\n"
+            + f"'{task_text}' -> NetBoxResponse <Success: True>\n"
+            + "-> The following edge ip-addresses fields had been updated:\n"
+            + f"{updated_fields}"
+        )
+    else:
+        results.append(
+            [
+                f"{task_result(text=task_text, changed=False, level_name='ERROR')}\n"
+                + f"'{task_text}' -> NetBoxResponse <Success: False>\n"
+                + f"-> Response Status Code: {resp.status_code}\n"
+                + f"-> Response Json:\n{resp.json()}"
+            ]
+        )
+
+    return results, failed
+
+
 def update_all_netbox_ip_addresses(prefix: dict) -> tuple:
     """
     Update all NetBox IP addresses VRF, Tenant, Tags and Location of a given prefix.
@@ -1045,28 +1229,29 @@ def main(nr_config: str, nmap_scan: bool = False, overwrite_active: list[str] =
         * Load active NetBox prefixes.
         * Load all non-container NetBox prefixes.
         * Load NetBox VLANs.
-        * Scan active NetBox prefixes with Nmap and update IP addresses status, DNS names, and ports.
-        * Update all IP addresses vrf, tenant, tags, and location.
-        * Update all VLANs status, tenant, tags, and location.
+        * Update OOB/T1/T2 prefixes IP-addresses status, DNS names, and ports with a Nmap scan
+        * Update all prefixes IP-addresses status and DNS names from IP-Fabric
+        * Update all active IP-addresses switch/switchport info from IP-Fabric
+        * Update all prefixes IP-addresses vrf, tenant, tags, and location
+        * Update all VLANs status, tenant, tags, and location
 
     * Exits:
         * Exits with code 1 if the Nornir configuration file is empty or if any task fails.
     """
 
-    #### Load NetBox Inventory ##############################################################################
+    #### Initialize Nornir ##################################################################################
 
-    task_text = "Load NetBox Inventory"
-    print_task_title(title=task_text)
+    # Initialize, transform and filter the Nornir inventory are return the filtered Nornir object
+    # Define data to load from NetBox in addition to the base Nornir inventory plugin
+    add_netbox_data = {"load_virtual_chassis_data": True}
+    nr = init_nornir(config_file=nr_config, add_netbox_data=add_netbox_data)
+    # Get the NetBox url from the inventory options
+    nb_url = nr.config.inventory.options["nb_url"]
 
-    # Load the Nornir yaml config file as dict and print a error message
-    nr_config_dict = load_yaml_file(
-        file=nr_config, text="Load Nornir Config File", silent=False, verbose=False
-    )
-    # Check the loaded config file and exit the script with exit code 1 if the dict is empty
-    if not nr_config_dict:
-        sys.exit(1)
-    # Get the NetBox URL (Authentication token will be loaded as nb_token env variable)
-    nb_url = nr_config_dict["inventory"]["options"]["nb_url"]
+    #### Load NetBox IPAM Data ##############################################################################
+
+    task_text = "Load NetBox IPAM Data"
+    print_task_title(title=task_text)
 
     # Load Active NetBox Prefixes from Tenant 'none', 'oob', 'tier-1' and 'tier-2' (except marked utilized)
     # These prefixes will be scanned with nmap and updated
@@ -1076,9 +1261,9 @@ def main(nr_config: str, nmap_scan: bool = False, overwrite_active: list[str] =
         query={"status": "active", "tenant": ["none", "oob", "tier-1", "tier-2"], "mark_utilized": "false"},
     )
 
-    # Load Active and Inventory NetBox Prefixes from all Tenants (except marked utilized)
+    # Load Active and NetBox Prefixes from all Tenants (except marked utilized)
     # These prefixes will be updated with input from IP-Fabric
-    nb_active_inventory_all_prefixes = load_netbox_data(
+    nb_active_all_prefixes = load_netbox_data(
         task_text="Load Active OOB/T1/T2/T3/T4 NetBox Prefixes",
         nb_api_url=f"{nb_url}/api/ipam/prefixes/",
         query={
@@ -1089,7 +1274,6 @@ def main(nr_config: str, nmap_scan: bool = False, overwrite_active: list[str] =
     )
 
     # Load NetBox Non-Container Prefixes
-    # Query "status__n" not working anymore since v4.1.3
     nb_subnet_prefixes = load_netbox_data(
         task_text="Load All Non-Container NetBox Prefixes",
         nb_api_url=f"{nb_url}/api/ipam/prefixes/",
@@ -1109,23 +1293,23 @@ def main(nr_config: str, nmap_scan: bool = False, overwrite_active: list[str] =
         # Set the task title
         title = "Nmap Scan and Update Active OOB/T1/T2 NetBox Prefixes IP-Addresses"
 
-        # Run the thread pool to update all NetBox IP-Addresses Status and DNS-Name
+        # Run the thread pool to update all NetBox IP-Addresses Status, DNS-Name and Open Ports
         # 1. arg is the input type ('nmap' or 'ip-fabric')
         # 2. arg is the list of active IP addresses to overwrite
         thread_result = run_thread_pool(
             title=title,
             task=update_netbox_prefix_ip_addresses,
             thread_list=nb_active_oob_t1_t2_prefixes,
-            max_workers=50,
-            args=("nmap", overwrite_active),
+            max_workers=50,  # NetBox have no rate limit, but 50 threads are a good tested value with Nmap
+            args=("nmap", overwrite_active, nr_config),
         )
         # Print the thread pool results and exit the script if any task has failed
         print_thread_pool_results(title=title, thread_result=thread_result, fail_hard=True)
 
-    #### IP-Fabric Update Active & Inventory NetBox Prefixes IP-Addresses ###################################
+    #### IP-Fabric Update Active NetBox Prefixes IP-Addresses ###############################################
 
     # Set the task title
-    title = "IP-Fabric Update Active OOB/T1/T2/T3/T4 NetBox Prefixes IP-Addresses"
+    title = "IP-Fabric Update All Active NetBox Prefixes IP-Addresses"
 
     # Run the thread pool to update all NetBox IP-Addresses Status and DNS-Name
     # 1. arg is the input type ('nmap' or 'ip-fabric')
@@ -1133,9 +1317,35 @@ def main(nr_config: str, nmap_scan: bool = False, overwrite_active: list[str] =
     thread_result = run_thread_pool(
         title=title,
         task=update_netbox_prefix_ip_addresses,
-        thread_list=nb_active_inventory_all_prefixes,
-        max_workers=5,
-        args=("ip-fabric", overwrite_active),
+        thread_list=nb_active_all_prefixes,
+        max_workers=5,  # Each thread will create the own Nornir object == Max. 5 concurrent Nornir logins
+        args=("ip-fabric", overwrite_active, nr_config),
+    )
+    # Print the thread pool results and exit the script if any task has failed
+    print_thread_pool_results(title=title, thread_result=thread_result, fail_hard=True)
+
+    #### IP-Fabric Update Switch & Switchport Info for IP-Addresses #########################################
+
+    # Set the task title
+    title = "IP-Fabric Update All Active NetBox IP-Addresses Switch/Switchport Info"
+
+    # Connect to IP-Fabric
+    ipf = IPFClient(
+        base_url=os.environ["IPF_URL"], auth=os.environ["IPF_TOKEN"], snapshot_id="$last", verify=False
+    )
+    # Get all mac-addresses on edge ports from the IP-Fabric technology mac table
+    filters = {"edge": ["eq", True]}
+    columns = ["hostname", "intName", "mac", "vendor"]
+    ipf_mac_table = ipf.technology.addressing.mac_table.all(filters=filters, columns=columns)
+    ipf_edge_mac_dict = {str(x["mac"]): x for x in ipf_mac_table}
+
+    # Run the thread pool to update all NetBox IP-Addresses Switch and Switchport Info
+    thread_result = run_thread_pool(
+        title=title,
+        task=update_netbox_ip_address_switchport,
+        thread_list=nb_active_all_prefixes,
+        max_workers=3,  # More then 3 threads will run into rate limit errors of IP-Fabric
+        args=(ipf_edge_mac_dict, nr),
     )
     # Print the thread pool results and exit the script if any task has failed
     print_thread_pool_results(title=title, thread_result=thread_result, fail_hard=True)

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/nornir_collection/nornir_plugins/inventory/netbox.py

@@ -111,6 +111,7 @@ class DSCNetBoxInventory:
         nb_url: Optional[str] = None,
         nb_token: Optional[str] = None,
         ssl_verify: Union[bool, str] = True,
+        nb_silent: bool = False,
         **kwargs: Any,
     ) -> None:
         """
@@ -118,12 +119,14 @@ class DSCNetBoxInventory:
         """
         nb_url = nb_url or os.environ.get("NB_URL")
         nb_token = nb_token or os.environ.get("NB_TOKEN")
+        nb_silent = nb_silent or os.environ.get("NB_SILENT")
 
         self.task_text = "NORNIR initialize inventory plugin DSCNetBoxInventory"
         self.nb_url = nb_url
         self.session = requests.Session()
         self.session.headers.update({"Authorization": f"Token {nb_token}"})
         self.session.verify = ssl_verify
+        self.nb_silent = nb_silent
 
     def _get_resources(self, url: str, params: Dict[str, Any] = {}) -> List[Dict[str, Any]]:
         """
@@ -154,7 +157,8 @@ class DSCNetBoxInventory:
         """
         yml = ruamel.yaml.YAML(typ="safe")
 
-        print_task_name(text=self.task_text)
+        if not self.nb_silent:
+            print_task_name(text=self.task_text)
 
         # Load all NetBox devices with a primary IP address assigned
         devices: List[Dict[str, Any]] = []
@@ -237,9 +241,10 @@ class DSCNetBoxInventory:
         for h in hosts.values():
             h.groups = ParentGroups([groups[g] for g in h.groups])
 
-        # Print the DSCNetBoxInventory result
-        print(task_info(text=self.task_text, changed=False))
-        print(f"'{self.task_text}' -> NornirResponse <Success: True>")
-        print(f"-> Loaded NetBox base inventory data of {len(hosts)} devices")
+        if not self.nb_silent:
+            # Print the DSCNetBoxInventory result
+            print(task_info(text=self.task_text, changed=False))
+            print(f"'{self.task_text}' -> NornirResponse <Success: True>")
+            print(f"-> Loaded NetBox base inventory data of {len(hosts)} devices")
 
         return Inventory(hosts=hosts, groups=groups, defaults=defaults)

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/nornir_collection/nornir_plugins/inventory/utils.py

@@ -34,14 +34,15 @@ from nornir_collection.utils import (
 #### Inventory Transform Functions ##########################################################################
 
 
-def nr_transform_host_creds_from_env(nr: Nornir) -> None:
+def nr_transform_host_creds_from_env(nr: Nornir, silent: bool = False) -> None:
     """
     This function loads the host login credentials from environment variables and stores them directly under
     the host inventory level. This function can be extended to to more host transformation.
     """
 
-    task_text = "NORNIR transform host credentials env variable"
-    print(task_name(text=task_text))
+    if not silent:
+        task_text = "NORNIR transform host credentials env variable"
+        print(task_name(text=task_text))
 
     # Recreate Nornir 2.x transform function behavior
     for host in nr.inventory.hosts.values():
@@ -98,9 +99,10 @@ def nr_transform_host_creds_from_env(nr: Nornir) -> None:
             msg="-> Analyse the error message and identify the root cause",
         )
 
-    print(task_info(text=task_text, changed=False))
-    print(f"'{task_text}' -> OS.EnvironResponse <Success: True>")
-    print(f"-> Transformed {len(nr.inventory.hosts)} host credentials env variables")
+    if not silent:
+        print(task_info(text=task_text, changed=False))
+        print(f"'{task_text}' -> OS.EnvironResponse <Success: True>")
+        print(f"-> Transformed {len(nr.inventory.hosts)} host credentials env variables")
 
 
 def _tuple_with_env_list(iterable: dict) -> tuple:
@@ -124,6 +126,7 @@ def nr_transform_inv_from_env(
     iterable: dict,
     env_mandatory: dict = False,
     verbose: bool = False,
+    silent: bool = False,
 ) -> None:
     """
     This function transforms all environment variables in the iterable. It loops over a nested dictionary and
@@ -148,8 +151,9 @@ def nr_transform_inv_from_env(
 
     # If the mandatory_transform_env_keys list it not empty -> Transform all env variables in the list
     if mandatory_transform_env_keys:
-        task_text = f"NORNIR transform mandatory {inv_type} env variable"
-        print(task_name(text=task_text))
+        if not silent:
+            task_text = f"NORNIR transform mandatory {inv_type} env variable"
+            print(task_name(text=task_text))
 
         # Verify that all mandatory env key-value pairs exists in the Nornir inventory
         for key, value in env_mandatory.items():
@@ -165,12 +169,13 @@ def nr_transform_inv_from_env(
                 # If the environ load fails a KeyError would be raised and catched by the exception
                 transform_env(iterable=iterable, startswith=env_key)
 
-            print(task_info(text=task_text, changed=False))
-            print(f"'{task_text}' -> OS.EnvironResponse <Success: True>")
-            print(f"-> Transformed {len(mandatory_transform_envs)} mandatory env variables")
-            if verbose:
-                for env in mandatory_transform_envs:
-                    print(f"  - {env}")
+            if not silent:
+                print(task_info(text=task_text, changed=False))
+                print(f"'{task_text}' -> OS.EnvironResponse <Success: True>")
+                print(f"-> Transformed {len(mandatory_transform_envs)} mandatory env variables")
+                if verbose:
+                    for env in mandatory_transform_envs:
+                        print(f"  - {env}")
 
         except KeyError as error:
             print(task_error(text=task_text, changed=False))
@@ -182,8 +187,9 @@ def nr_transform_inv_from_env(
 
     # If the transform_env_keys list it not empty -> Transform all env variables in the list
     if transform_envs:
-        task_text = f"NORNIR transform non-mandatory {inv_type} env variable"
-        print(task_name(text=task_text))
+        if not silent:
+            task_text = f"NORNIR transform non-mandatory {inv_type} env variable"
+            print(task_name(text=task_text))
 
         try:
             # Loop over the generator object items and add the matching elemens based on the key to a list
@@ -191,12 +197,13 @@ def nr_transform_inv_from_env(
                 # If the environ load fails a KeyError would be raised and catched by the exception
                 transform_env(iterable=iterable, startswith=env_key)
 
-            print(task_info(text=task_text, changed=False))
-            print(f"'{task_text}' -> OS.EnvironResponse <Success: True>")
-            print(f"-> Transformed {len(transform_envs)} non-mandatory env variables")
-            if verbose:
-                for env in transform_envs:
-                    print(f"  - {env}")
+            if not silent:
+                print(task_info(text=task_text, changed=False))
+                print(f"'{task_text}' -> OS.EnvironResponse <Success: True>")
+                print(f"-> Transformed {len(transform_envs)} non-mandatory env variables")
+                if verbose:
+                    for env in transform_envs:
+                        print(f"  - {env}")
 
         except KeyError as error:
             print(task_error(text=task_text, changed=False))
@@ -213,6 +220,7 @@ def init_nornir(
     env_mandatory: dict = False,
     args: argparse.Namespace = False,
     add_netbox_data: dict[str:bool] = None,
+    silent: bool = False,
 ) -> Nornir:
     """
     The Nornir inventory will be initialized, the host username and password will be transformed and loaded
@@ -221,8 +229,8 @@ def init_nornir(
     are defined in the inventory. The function returns a filtered Nornir object or quits with an error message
     in case of issues during the function.
     """
-
-    print_task_title("Initialize Nornir Inventory")
+    if not silent:
+        print_task_title("Initialize Nornir Inventory")
 
     # Set a timer to track how long the init_nornir functions takes to finish
     timer_start = time.time()
@@ -236,14 +244,17 @@ def init_nornir(
     load_dotenv(".env", override=True)
 
     # Initialize Nornir Object with a config file
+    # Set and unset the NB_SILENT environment variable to control the Nornir output of the inventory plugin
+    os.environ["NB_SILENT"] = str(silent)
     nr = InitNornir(config_file=config_file)
+    os.environ.pop("NB_SILENT", None)
 
     # Load additional data from NetBox into the Nornir inventory
     if add_netbox_data:
-        load_additional_netbox_data(nr=nr, add_netbox_data=add_netbox_data)
+        load_additional_netbox_data(nr=nr, add_netbox_data=add_netbox_data, silent=silent)
 
     # Transform the host username and password from environment variables
-    nr_transform_host_creds_from_env(nr=nr)
+    nr_transform_host_creds_from_env(nr=nr, silent=silent)
 
     # Transform the inventory and load all env variables staring with "env_" in the defaults yaml files
     nr_transform_inv_from_env(
@@ -251,6 +262,7 @@ def init_nornir(
         iterable=nr.inventory.defaults.data,
         verbose=args.verbose if args else False,
         env_mandatory=env_mandatory,
+        silent=silent,
     )
 
     # Transform the inventory and load all env variables staring with "env_" in the groups yaml files
@@ -259,17 +271,19 @@ def init_nornir(
             inv_type="groups",
             iterable=nr.inventory.groups[group].data,
             verbose=args.verbose if args else False,
+            silent=silent,
         )
 
     # Filter the Nornir inventory based on the provided arguments from argparse
     if args:
-        nr = nr_filter_args(nr=nr, args=args)
-
-    # Print the timer result
-    task_text = "NORNIR initialize timer"
-    print(f"{task_name(text=task_text)}")
-    print(task_info(text=task_text, changed=False))
-    print(f"'{task_text}' -> NornirResponse <Success: True>")
-    print(f"-> Initialize Nornir finished in: {round(time.time() - timer_start, 1)}s")
+        nr = nr_filter_args(nr=nr, args=args, silent=silent)
+
+    if not silent:
+        # Print the timer result
+        task_text = "NORNIR initialize timer"
+        print(f"{task_name(text=task_text)}")
+        print(task_info(text=task_text, changed=False))
+        print(f"'{task_text}' -> NornirResponse <Success: True>")
+        print(f"-> Initialize Nornir finished in: {round(time.time() - timer_start, 1)}s")
 
     return nr

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/nornir_collection/utils.py

@@ -430,20 +430,20 @@ def nr_filter_by_hosts(nr: Nornir, hosts: Union[str, list], task_text: str = Non
     return nr_filtered
 
 
-def nr_filter_args(nr: Nornir, args: argparse.Namespace) -> Nornir:
+def nr_filter_args(nr: Nornir, args: argparse.Namespace, silent: bool = False) -> Nornir:
     """
     This function filters the Nornir inventory with a tag or a host argument provided by argparse. Prior
     Argparse validation needs to ensure that only one argument is present and that the tag or host argument
     creates a correct inventory filtering will be verified. The new filtered Nornir object will be returned
     or the script terminates with an error message.
     """
-
-    task_text = "NORNIR filter inventory"
-    print_task_name(task_text)
+    if not silent:
+        task_text = "NORNIR filter inventory"
+        print_task_name(task_text)
 
     # If the --hosts argument is set, verify that the host exist
     if hasattr(args, "hosts"):
-        nr = nr_filter_by_hosts(nr=nr, hosts=args.hosts, silent=False)
+        nr = nr_filter_by_hosts(nr=nr, hosts=args.hosts, silent=silent)
 
         return nr
 
@@ -451,7 +451,7 @@ def nr_filter_args(nr: Nornir, args: argparse.Namespace) -> Nornir:
     if hasattr(args, "role") or hasattr(args, "tags"):
         role = args.role if hasattr(args, "role") else None
         tags = args.tags.split(",") if hasattr(args, "tags") else None
-        nr = nr_filter_by_role_and_tag(nr=nr, role=role, tags=tags, silent=False)
+        nr = nr_filter_by_role_and_tag(nr=nr, role=role, tags=tags, silent=silent)
 
         return nr
 

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/nornir_collection.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nornir-collection
-Version: 0.0.27
+Version: 0.0.29
 Summary: Nornir-Collection contains network automation functions and complete IaC workflows with         Nornir and other python libraries. It contains Nornir tasks and general functions in Nornir style.
 Author: Willi Kubny
 Author-email: willi.kubny@gmail.ch
@@ -10,19 +10,19 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: nornir==3.3.0
+Requires-Dist: nornir==3.5.0
 Requires-Dist: nornir-jinja2==0.2.0
-Requires-Dist: nornir-salt==0.22.2
+Requires-Dist: nornir-salt==0.22.5
 Requires-Dist: nornir-netmiko==1.0.1
-Requires-Dist: nornir-scrapli==2023.7.30
-Requires-Dist: scrapli-netconf==2023.7.30
+Requires-Dist: nornir-scrapli==2024.7.30
+Requires-Dist: scrapli-netconf==2024.1.30
+Requires-Dist: nornir-utils==0.2.0
+Requires-Dist: nornir-pyfgt==1.0.3
 Requires-Dist: scrapli[ssh2]
 Requires-Dist: scrapli[genie]
-Requires-Dist: nornir-utils==0.2.0
 Requires-Dist: pynetbox==7.4.0
-Requires-Dist: ipfabric==7.0.2
+Requires-Dist: ipfabric==7.2.2
 Requires-Dist: py-pure-client==1.51.0
-Requires-Dist: nornir-pyfgt==1.0.3
 Requires-Dist: python-nmap==0.7.1
 Requires-Dist: pybatfish==2023.12.16.1270
 Requires-Dist: requests==2.31.0
@@ -30,7 +30,7 @@ Requires-Dist: pre-commit==3.6.0
 Requires-Dist: pyyaml==6.0.1
 Requires-Dist: pyfiglet==1.0.2
 Requires-Dist: GitPython==3.1.40
-Requires-Dist: python-dotenv==1.0.0
+Requires-Dist: python-dotenv==1.1.1
 Requires-Dist: beautifultable==1.1.0
 Requires-Dist: pandas==2.2.2
 Requires-Dist: openpyxl==3.1.2

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/nornir_collection.egg-info/requires.txt

@@ -1,16 +1,16 @@
-nornir==3.3.0
+nornir==3.5.0
 nornir-jinja2==0.2.0
-nornir-salt==0.22.2
+nornir-salt==0.22.5
 nornir-netmiko==1.0.1
-nornir-scrapli==2023.7.30
-scrapli-netconf==2023.7.30
+nornir-scrapli==2024.7.30
+scrapli-netconf==2024.1.30
+nornir-utils==0.2.0
+nornir-pyfgt==1.0.3
 scrapli[ssh2]
 scrapli[genie]
-nornir-utils==0.2.0
 pynetbox==7.4.0
-ipfabric==7.0.2
+ipfabric==7.2.2
 py-pure-client==1.51.0
-nornir-pyfgt==1.0.3
 python-nmap==0.7.1
 pybatfish==2023.12.16.1270
 requests==2.31.0
@@ -18,7 +18,7 @@ pre-commit==3.6.0
 pyyaml==6.0.1
 pyfiglet==1.0.2
 GitPython==3.1.40
-python-dotenv==1.0.0
+python-dotenv==1.1.1
 beautifultable==1.1.0
 pandas==2.2.2
 openpyxl==3.1.2

{nornir_collection-0.0.27 → nornir_collection-0.0.29}/setup.py

@@ -11,7 +11,7 @@ with open("requirements.txt", encoding="utf-8") as f:
 
 setuptools.setup(
     name="nornir-collection",
-    version="0.0.27",
+    version="0.0.29",
     author="Willi Kubny",
     author_email="willi.kubny@gmail.ch",
     description="Nornir-Collection contains network automation functions and complete IaC workflows with \