nornir-collection 0.0.18__tar.gz → 0.0.20__tar.gz

{nornir_collection-0.0.18 → nornir_collection-0.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nornir-collection
-Version: 0.0.18
+Version: 0.0.20
 Summary: Nornir-Collection contains network automation functions and complete IaC workflows with         Nornir and other python libraries. It contains Nornir tasks and general functions in Nornir style.
 Author: Willi Kubny
 Author-email: willi.kubny@gmail.ch

{nornir_collection-0.0.18 → nornir_collection-0.0.20}/nornir_collection/netbox/update_prefixes_ip_addresses.py

@@ -156,27 +156,27 @@ def create_nb_response_result(
                     ports = (
                         ", ".join([str(p) for p in ip["ports"]]) if "ports" in ip and ip["ports"] else "None"
                     )
-                    updated_fields.append(f"- {address} (DNS: {dns_name}, Ports: {ports})")
+                    updated_fields.append(f"  - {address} (DNS: {dns_name}, Ports: {ports})")
                 else:
-                    updated_fields.append(f"- {address} (DNS: {dns_name})")
+                    updated_fields.append(f"  - {address} (DNS: {dns_name})")
         # If data is a dictionary containing the prefix or information vlan associated with a prefix
         elif nb_type in ("ip", "vlan") and isinstance(data, dict):
             # If the response json contains the key 'vid' the response is from a vlan and has no VRF
             if "vid" in resp.json():
-                updated_fields.append(f"- Status: {data['status']['label']}")
+                updated_fields.append(f"  - Status: {data['status']['label']}")
             # It's the response from a ip-address and ha a VRF
             else:
-                updated_fields.append(f"- VRF: {data['vrf'] if data['vrf'] else 'Global'}")
+                updated_fields.append(f"  - VRF: {data['vrf'] if data['vrf'] else 'Global'}")
             updated_fields.extend(
                 [
-                    f"- Tenant: {data['tenant']['name'] if data['tenant'] else 'None'}",
-                    f"- Tags: {(', '.join([i['name'] for i in data['tags']])) or 'None'}",
-                    f"- Location: {', '.join(list(data['custom_fields']['ipam_location'])).upper() or 'None'}",  # noqa: E501
+                    f"  - Tenant: {data['tenant']['name'] if data['tenant'] else 'None'}",
+                    f"  - Tags: {(', '.join([i['name'] for i in data['tags']])) or 'None'}",
+                    f"  - Location: {', '.join(list(data['custom_fields']['ipam_location'])).upper() or 'None'}",  # noqa: E501
                 ]
             )
         # If the type is 'vlan' and data is empty
         elif nb_type == "vlan" and not data:
-            updated_fields.append("- Tags: L2 Only")
+            updated_fields.append("  - Tags: L2 Only")
         else:
             result = [
                 f"{task_result(text=task_text, changed=True, level_name='ERROR')}\n"
@@ -266,23 +266,33 @@ def create_nb_ip_payload(
     return payload
 
 
-def nmap_scan_prefix(prefix: dict, result: list) -> tuple:
+def nmap_scan_host_ip_or_subnet(hosts: str) -> list:
     """
     TBD
     """
+    # Create an ip-interface object from the hosts argument.
+    # Can be a host ip-address with CIDR netmask or a network prefix with CIDR netmask
+    ip_iface = ipaddress.ip_interface(hosts)
     # Get the prefix length from the prefix
-    prefixlen = ipaddress.ip_network(prefix["prefix"]).prefixlen
-    # Get the network and broadcast address of the prefix to exclude them from the nmap scan
-    network_addr = ipaddress.ip_network(prefix["prefix"]).network_address
-    broadcast_addr = ipaddress.ip_network(prefix["prefix"]).broadcast_address
+    prefixlen = ip_iface.network.prefixlen
     # Set the nmap scan arguments
-    arguments = f"-PE -PP -PA21 -PS80,443,3389 -PU161,40125 --source-port 53 --exclude {network_addr},{broadcast_addr}"  # noqa E501
+    arguments = "-PE -PP -PA21 -PS80,443,3389 -PU161,40125 --source-port 53"
+
+    # If the ip-address is the network address of the prefix, then it's a whole prefix to scan
+    if ip_iface.ip == ip_iface.network.network_address:
+        # Add the the nmap arguments that the network and broadcast address should be excluded from the scan
+        arguments += f" --exclude {ip_iface.network.network_address},{ip_iface.network.broadcast_address}"
+        # Scan the whole prefix
+        scan_target = str(ip_iface.network)
+    else:
+        # Scan only the host ip-address
+        scan_target = str(ip_iface.ip)
 
-    # Nmap ARP scan for the prefix and add a list of active ip-addresses and other details to prefix
+    # Nmap ARP scan and add a list of active ip-addresses and other details to the list
     nm = nmap.PortScanner()
-    nm.scan(hosts=prefix["prefix"], arguments=arguments, sudo=True)
+    nm.scan(hosts=scan_target, arguments=arguments, sudo=True)
     if nm.all_hosts():
-        prefix["datasource_ips"] = [
+        nmap_scan_result = [
             {
                 "address": f"{nm[host]['addresses']['ipv4']}/{prefixlen}",
                 "dns_name": nm[host]["hostnames"][0]["name"],
@@ -291,23 +301,19 @@ def nmap_scan_prefix(prefix: dict, result: list) -> tuple:
             for host in nm.all_hosts()
         ]
     else:
-        prefix["datasource_ips"] = []
+        nmap_scan_result = []
 
-    # Print the task result
-    text = f"Nmap Scan Prefix {prefix['prefix']} for active IP-Addresses"
-    result.append(
-        f"{task_result(text=text, changed=False, level_name='INFO')}\n"
-        + f"'{text}' -> NetBoxResponse <Success: True>\n"
-        + f"-> Nmap prefix scan ip-address count: {len(prefix['datasource_ips'])}"
-    )
+    return nmap_scan_result
 
-    return result, prefix
 
-
-def get_ipfabric_data_for_prefix(ipf: IPFClient, prefix: dict, result: list) -> tuple:
+def get_ipfabric_data_for_prefix(prefix: dict) -> tuple:
     """
     TBD
     """
+    # Connect to IP-Fabric
+    ipf = IPFClient(
+        base_url=os.environ["IPF_URL"], auth=os.environ["IPF_TOKEN"], snapshot_id="$last", verify=False
+    )
     # Get the prefix length from the prefix
     prefixlen = ipaddress.ip_network(prefix["prefix"]).prefixlen
     # Get all ip-addresses of the prefix from the IP-Fabric technology arp table
@@ -315,6 +321,7 @@ def get_ipfabric_data_for_prefix(ipf: IPFClient, prefix: dict, result: list) ->
     filters = {"ip": ["cidr", prefix["prefix"]], "proxy": ["eq", False]}
     arp_ip_list = list(set([x["ip"] for x in ipf.technology.addressing.arp_table.all(filters=filters)]))
     # Get all ip-addresses of the prefix from the IP-Fabric inventory managed ipv4 table
+    filters = {"ip": ["cidr", prefix["prefix"]]}
     managed_ipv4_ip_list = list(
         set([x["ip"] for x in ipf.technology.addressing.managed_ip_ipv4.all(filters=filters, columns=["ip"])])
     )
@@ -334,7 +341,7 @@ def get_ipfabric_data_for_prefix(ipf: IPFClient, prefix: dict, result: list) ->
     # Combine all ip-addresses lists and remove duplicates
     all_ips = list(set(arp_ip_list + managed_ipv4_ip_list + interface_ip_list + host_ip_list))
     # Add a list of all ip-addresses and other details to prefix
-    prefix["datasource_ips"] = [
+    ipf_ip_address_result = [
         {
             "address": f"{ip}/{prefixlen}",
             "dns_name": host_ip_dict[ip] if ip in host_ip_dict.keys() else "",
@@ -343,15 +350,7 @@ def get_ipfabric_data_for_prefix(ipf: IPFClient, prefix: dict, result: list) ->
         for ip in all_ips
     ]
 
-    # Print the task result
-    text = f"IP-Fabric Get Data for Prefix {prefix['prefix']} IP-Addresses"
-    result.append(
-        f"{task_result(text=text, changed=False, level_name='INFO')}\n"
-        + f"'{text}' -> IPFResponse <Success: True>\n"
-        + f"-> IP-Fabric prefix ip-address count: {len(prefix['datasource_ips'])}"
-    )
-
-    return result, prefix
+    return ipf_ip_address_result
 
 
 def get_nb_ips_and_external_datasource(nb_url: str, prefix: dict, ds: Literal["nmap", "ip-fabric"]) -> tuple:
@@ -374,14 +373,26 @@ def get_nb_ips_and_external_datasource(nb_url: str, prefix: dict, ds: Literal["n
     # Get the ip-addresses of the prefix from the datasource
     if ds == "nmap":
         # Scan the prefix with nmap
-        result, prefix = nmap_scan_prefix(prefix=prefix, result=result)
-    elif ds == "ip-fabric":
-        # Connect to IP-Fabric
-        ipf = IPFClient(
-            base_url=os.environ["IPF_URL"], auth=os.environ["IPF_TOKEN"], snapshot_id="$last", verify=False
+        prefix["datasource_ips"] = nmap_scan_host_ip_or_subnet(hosts=prefix["prefix"])
+
+        # Print the task result
+        text = f"Nmap Scan Prefix {prefix['prefix']} for active IP-Addresses"
+        result.append(
+            f"{task_result(text=text, changed=False, level_name='INFO')}\n"
+            + f"'{text}' -> NetBoxResponse <Success: True>\n"
+            + f"-> Nmap prefix scan ip-address count: {len(prefix['datasource_ips'])}"
         )
+    elif ds == "ip-fabric":
         # Get the ip-addresses from the IP-Fabric
-        result, prefix = get_ipfabric_data_for_prefix(ipf=ipf, prefix=prefix, result=result)
+        prefix["datasource_ips"] = get_ipfabric_data_for_prefix(prefix=prefix)
+
+        # Print the task result
+        text = f"IP-Fabric Get Data for Prefix {prefix['prefix']} IP-Addresses"
+        result.append(
+            f"{task_result(text=text, changed=False, level_name='INFO')}\n"
+            + f"'{text}' -> IPFResponse <Success: True>\n"
+            + f"-> IP-Fabric prefix ip-address count: {len(prefix['datasource_ips'])}"
+        )
     else:
         # Invalid datasource
         failed = True
@@ -512,7 +523,7 @@ def update_discovered_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap"
         result.extend(sub_result)
         failed = True if sub_failed else failed
 
-    # Update the ip-addresses with the status 'auto_discovered' that are part of the nmap scan list
+    # Update the ip-addresses with the status 'auto_discovered' that are part of the datasource list
     update_ips = create_ip_list(
         loop_list=prefix["discovered_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
@@ -556,10 +567,19 @@ def delete_inactive_auto_discovered_ip_addresses(
     failed = False
     task_text = "Delete Auto-Discovered IP-Addresses"
 
-    # Delete the ip-addresses with the status 'auto_discovered' that are not in the nmap scan list
-    delete_ips = create_ip_list(
+    # Delete the ip-addresses with the status 'auto_discovered' that are not in the datasource list
+    maybe_delete_ips = create_ip_list(
         loop_list=prefix["discovered_ips"], check_list=prefix["datasource_ips"], is_in_both=False
     )
+    delete_ips = []
+    if maybe_delete_ips:
+        # Nmap scan ip-addresses of the maybe_delete_ips list
+        for ip in maybe_delete_ips:
+            # Scan the prefix with nmap
+            scan_result = nmap_scan_host_ip_or_subnet(hosts=ip["address"])
+            # Add the nmal scan result to the inactive_ips list
+            if scan_result:
+                delete_ips.extend(scan_result)
 
     # If ip-addresses have been found
     if delete_ips:
@@ -593,7 +613,7 @@ def update_reserved_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap",
     failed = False
     task_text = "Update Reserved IP-Addresses Status"
 
-    # Update the ip-addresses with the status 'reserved' that are part of the nmap scan list
+    # Update the ip-addresses with the status 'reserved' that are part of the datacenter list
     update_ips = create_ip_list(
         loop_list=prefix["reserved_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
@@ -631,17 +651,27 @@ def update_inactive_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap",
     failed = False
     task_text = "Update Inactive IP-Addresses Status"
 
-    # Update the ip-addresses with the status 'inactive' that are part of the nmap scan list
-    inactive_ips = create_ip_list(
+    # Update the ip-addresses with the status 'inactive' that are part of the datasource list
+    active_ips = create_ip_list(
         loop_list=prefix["inactive_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
+    # Double-check ip-addresses with the status 'inactive' that are not part of the datasource list
+    maybe_active_ips = create_ip_list(
+        loop_list=prefix["inactive_ips"], check_list=prefix["datasource_ips"], is_in_both=False
+    )
+    if maybe_active_ips:
+        # Nmap scan ip-addresses of the maybe_inactive_ips list
+        for ip in maybe_active_ips:
+            # Scan the prefix with nmap
+            scan_result = nmap_scan_host_ip_or_subnet(hosts=ip["address"])
+            # Add the nmal scan result to the inactive_ips list
+            if scan_result:
+                active_ips.extend(scan_result)
 
     # If ip-addresses have been found
-    if inactive_ips:
+    if active_ips:
         # Create the payload to update the ip-addresses
-        payload = create_nb_ip_payload(
-            parent_prefix=prefix, data=inactive_ips, ds=ds, desired_status="active"
-        )
+        payload = create_nb_ip_payload(parent_prefix=prefix, data=active_ips, ds=ds, desired_status="active")
         # PATCH request to update the ip-addresses
         resp = patch_nb_resources(url=f"{nb_url}/api/ipam/ip-addresses/", payload=payload)
 
@@ -649,7 +679,7 @@ def update_inactive_ip_addresses(nb_url: str, prefix: dict, ds: Literal["nmap",
         text = "The following 'Inactive' ip-addresses had been set to status 'Active':"
         # The function returns the result list and True if the response is successful else False
         result, failed = create_nb_response_result(
-            resp=resp, nb_type="ip", data=inactive_ips, ds=ds, task_text=task_text, text=text
+            resp=resp, nb_type="ip", data=active_ips, ds=ds, task_text=task_text, text=text
         )
 
     return result, failed
@@ -673,7 +703,7 @@ def update_active_ip_addresses(
     failed = False
     task_text = "Update Active IP-Addresses Status"
 
-    # Update the ip-addresses with the status 'active' that are part of the nmap scan list
+    # Update the ip-addresses with the status 'active' that are part of the datacenter list
     active_ips = create_ip_list(
         loop_list=prefix["active_ips"], check_list=prefix["datasource_ips"], is_in_both=True
     )
@@ -694,10 +724,19 @@ def update_active_ip_addresses(
         result.extend(sub_result)
         failed = True if sub_failed else failed
 
-    # Update the ip-addresses with the status 'active' that are not part of the nmap scan list
-    inactive_ips = create_ip_list(
+    # Update the ip-addresses with the status 'active' that are not part of the datacenter list
+    maybe_inactive_ips = create_ip_list(
         loop_list=prefix["active_ips"], check_list=prefix["datasource_ips"], is_in_both=False
     )
+    inactive_ips = []
+    if maybe_inactive_ips:
+        # Nmap scan ip-addresses of the maybe_inactive_ips list
+        for ip in maybe_inactive_ips:
+            # Scan the prefix with nmap
+            scan_result = nmap_scan_host_ip_or_subnet(hosts=ip["address"])
+            # Add the nmal scan result to the inactive_ips list
+            if scan_result:
+                inactive_ips.extend(scan_result)
     # Create a new list to exclude the overwrite_active ip-addresses
     inactive_ips = [ip for ip in inactive_ips if ip["address"] not in overwrite_active]
 
@@ -784,7 +823,7 @@ def update_netbox_prefix_ip_addresses(prefix: list, *args) -> tuple:
     result, sub_failed = update_discovered_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
-    # Delete inactive 'auto-discovered' ip-addresses
+    # Delete inactive 'auto-discovered' ip-addresses (double check with partial nmap scan)
     result, sub_failed = delete_inactive_auto_discovered_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
@@ -792,11 +831,11 @@ def update_netbox_prefix_ip_addresses(prefix: list, *args) -> tuple:
     result, sub_failed = update_reserved_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
-    # Update 'inactive' ip-addresses -> set status to 'active'
+    # Update 'inactive' ip-addresses -> set status to 'active' (double check with partial nmap scan)
     result, sub_failed = update_inactive_ip_addresses(nb_url=nb_url, prefix=prefix, ds=ds)
     results, changed, failed = set_results_changed_failed(results, result, changed, sub_failed, failed)
 
-    # Update 'active' ip-addresses -> set status to 'active' or 'inactive'
+    # Update 'active' ip-addresses -> set status to 'active' or 'inactive' (check with partial nmap scan)
     result, sub_failed = update_active_ip_addresses(
         nb_url=nb_url, prefix=prefix, overwrite_active=overwrite_active, ds=ds
     )
@@ -1037,7 +1076,7 @@ def main(nr_config: str, nmap_scan: bool = False, overwrite_active: list[str] =
 
     # Load Active and Inventory NetBox Prefixes from all Tenants (except marked utilized)
     # These prefixes will be updated with input from IP-Fabric
-    nb_active_inventory_t3_t4_prefixes = load_netbox_data(
+    nb_active_inventory_all_prefixes = load_netbox_data(
         task_text="Load Active OOB/T1/T2/T3/T4 NetBox Prefixes",
         nb_api_url=f"{nb_url}/api/ipam/prefixes/",
         query={
@@ -1092,7 +1131,7 @@ def main(nr_config: str, nmap_scan: bool = False, overwrite_active: list[str] =
     thread_result = run_thread_pool(
         title=title,
         task=update_netbox_prefix_ip_addresses,
-        thread_list=nb_active_inventory_t3_t4_prefixes,
+        thread_list=nb_active_inventory_all_prefixes,
         max_workers=5,
         args=("ip-fabric", overwrite_active),
     )

{nornir_collection-0.0.18 → nornir_collection-0.0.20}/nornir_collection.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nornir-collection
-Version: 0.0.18
+Version: 0.0.20
 Summary: Nornir-Collection contains network automation functions and complete IaC workflows with         Nornir and other python libraries. It contains Nornir tasks and general functions in Nornir style.
 Author: Willi Kubny
 Author-email: willi.kubny@gmail.ch

{nornir_collection-0.0.18 → nornir_collection-0.0.20}/setup.py

@@ -11,7 +11,7 @@ with open("requirements.txt", encoding="utf-8") as f:
 
 setuptools.setup(
     name="nornir-collection",
-    version="0.0.18",
+    version="0.0.20",
     author="Willi Kubny",
     author_email="willi.kubny@gmail.ch",
     description="Nornir-Collection contains network automation functions and complete IaC workflows with \