ipfabric_netbox 4.3.2b4__tar.gz → 4.3.2b6__tar.gz

{ipfabric_netbox-4.3.2b4 → ipfabric_netbox-4.3.2b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipfabric_netbox
-Version: 4.3.2b4
+Version: 4.3.2b6
 Summary: NetBox plugin to sync IP Fabric data into NetBox
 License: MIT
 Keywords: netbox,ipfabric,plugin,sync

{ipfabric_netbox-4.3.2b4 → ipfabric_netbox-4.3.2b6}/ipfabric_netbox/__init__.py

@@ -6,7 +6,7 @@ class NetboxIPFabricConfig(PluginConfig):
     name = "ipfabric_netbox"
     verbose_name = "NetBox IP Fabric SoT Plugin"
     description = "Sync IP Fabric into NetBox"
-    version = "4.3.2b4"
+    version = "4.3.2b6"
     base_url = "ipfabric"
     min_version = "4.4.0"
 

{ipfabric_netbox-4.3.2b4 → ipfabric_netbox-4.3.2b6}/ipfabric_netbox/exceptions.py

@@ -44,3 +44,7 @@ class IPAddressDuplicateError(IngestionIssue, SyncError):
 class IPAddressPrimaryRemovalError(IngestionIssue, SyncError):
     def __str__(self):
         return "Error removing primary IP from other device."
+
+
+class RequiredDependencyFailedSkip(SearchError):
+    """Raised when a required dependency failed, causing this item to be skipped."""

{ipfabric_netbox-4.3.2b4 → ipfabric_netbox-4.3.2b6}/ipfabric_netbox/utilities/ipfutils.py

@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
 from typing import TypeVar
 
 from core.exceptions import SyncError
+from core.signals import clear_events
 from dcim.models import Device
 from django.conf import settings
 from django.core.exceptions import MultipleObjectsReturned
@@ -19,15 +20,18 @@ from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import Model
 from django.utils.text import slugify
 from django_tables2 import Column
+from extras.events import flush_events
 from ipfabric import IPFClient
 from jinja2.sandbox import SandboxedEnvironment
 from netbox.config import get_config
+from netbox.context import events_queue
 from netutils.utils import jinja2_convenience_function
 
 from ..choices import IPFabricSourceTypeChoices
 from ..choices import IPFabricSyncStatusChoices
 from ..exceptions import IPAddressDuplicateError
 from ..exceptions import IPAddressPrimaryRemovalError
+from ..exceptions import RequiredDependencyFailedSkip
 from ..exceptions import SearchError
 from ..exceptions import SyncDataError
 
@@ -37,6 +41,7 @@ if TYPE_CHECKING:
     from ..models import IPFabricIngestionIssue
     from ipam.models import IPAddress
     from ..models import IPFabricTransformMap
+    from ..models import IPFabricSync
 
 logger = logging.getLogger("ipfabric_netbox.utilities.ipf_utils")
 
@@ -72,6 +77,49 @@ def render_jinja2(template_code, context):
     return environment.from_string(source=template_code).render(**context)
 
 
+class EventsClearer:
+    """
+    Handles clearing of events after defined number of objects are saved to reduce memory usage.
+    ChangeLoggingMiddleware causes rest_framework Fields to be held in memory for the whole
+    duration of current request. This causes high memory usage during sync jobs.
+
+    The data is already written in DB due to branching ChangeDiff. So we can trigger
+    the events clearing here to free up memory.
+
+    For performance reasons we don't want to clear the event cache after every object is saved,
+    but we also don't want to wait until the end of the sync when syncing large number of objects.
+    This class helps to clear events after a defined threshold is reached.
+
+    Without it, the memory requirements are roughly 1,5GB per 10k changes.
+    """
+
+    def __init__(self, sender: object, threshold: int = 100) -> None:
+        self.sender = sender
+        self.threshold = threshold
+        self.counter = 0
+
+    def increment(self) -> None:
+        """
+        Increment the counter and clear events if threshold is reached.
+
+        This should not be called after every instance.snapshot() but instead
+        when the instance is processed. This makes sure it's changes are synced
+        as a whole. Calling it after every single snapshot() causes issues.
+        """
+        self.counter += 1
+        if self.counter >= self.threshold:
+            self.clear()
+
+    def clear(self) -> None:
+        logger.debug("Clearing events to reduce memory usage.")
+        # This makes sure webhooks are sent properly
+        if events := list(events_queue.get().values()):
+            flush_events(events)
+        # And now the queue can be cleared
+        clear_events.send(sender=None)
+        self.counter = 0
+
+
 class IPFabric(object):
     def __init__(self, parameters=None) -> None:
         if parameters:
@@ -325,14 +373,21 @@ def order_members(members: list[dict]) -> dict[str, list[dict]]:
     return devices
 
 
-def order_devices(
+def prepare_devices(
     devices: list[dict], members: dict[str, list[dict]]
 ) -> tuple[list[dict], list[dict]]:
+    """
+    Prepare devices list for syncing:
+     - incorporating virtual chassis members
+     - handle duplicate hostnames
+    """
+
     hostnames = [d["hostname"] for d in devices]
     counter = Counter(hostnames)
 
-    new_devices = []
-    ordered_devices = []
+    # All devices to be synced
+    all_devices = []
+    # All virtual chassis to be synced
     virtualchassis = []
 
     for device in devices[:]:
@@ -341,32 +396,33 @@ def order_devices(
         if child_members := members.get(device.get("sn")):
             # This device is the VC master, and we're iterating over all it's members
             for child_member in child_members:
+                # There is physically no device with hostname matching the virtual chassis
+                # There are only members, so "hostname/1", "hostname/2", etc.
+                new_device = deepcopy(device)
+                new_device[
+                    "hostname"
+                ] = f"{device['hostname']}/{child_member.get('member')}"
+                new_device["virtual_chassis"] = child_member
                 if device.get("sn") != child_member.get("memberSn"):
                     # VC members (non-master) are not in Device table, need to add them as new Device
-                    new_device = deepcopy(device)
-                    new_device[
-                        "hostname"
-                    ] = f"{device['hostname']}/{child_member.get('member')}"
                     new_device["model"] = child_member.get("pn")
                     new_device["sn"] = child_member.get("memberSn")
-                    new_device["virtual_chassis"] = child_member
-                    new_devices.append(new_device)
+                    all_devices.append(new_device)
                 else:
-                    # This is the master device
-                    device["virtual_chassis"] = child_member
-                virtualchassis.append(child_member)
+                    # Master device, create the virtual chassis based on it
+                    virtualchassis.append(child_member)
+                    all_devices.append(new_device)
             hostnames = [d["hostname"] for d in devices]
             counter = Counter(hostnames)
         else:
-            ordered_devices.append(device)
-    ordered_devices.extend(new_devices)
-    return ordered_devices, virtualchassis
+            all_devices.append(device)
+    return all_devices, virtualchassis
 
 
 class IPFabricSyncRunner(object):
     def __init__(
         self,
-        sync,
+        sync: "IPFabricSync",
         client: IPFClient = None,
         ingestion=None,
         settings: dict = None,
@@ -387,6 +443,8 @@ class IPFabricSyncRunner(object):
         # For now store only serial numbers since that is the largest dependency chain
         self.error_serials = set()
 
+        self.events_clearer = EventsClearer(sender=self.__class__, threshold=100)
+
     @staticmethod
     def get_error_serial(context: dict | None, data: dict | None) -> str | None:
         """Get error serial from context or raw data for skipping purposes."""
@@ -413,7 +471,8 @@ class IPFabricSyncRunner(object):
         context = context or {}
 
         error_serial = self.get_error_serial(context, data)
-        if error_serial:
+        # Ignore models that do not have any dependencies by serial number
+        if error_serial and model not in ["ipaddress", "macaddress"]:
             self.error_serials.add(error_serial)
 
         # TODO: This is to prevent circular import issues, clean it up later.
@@ -516,11 +575,21 @@ class IPFabricSyncRunner(object):
         # First check if there are any previous errors linked to this object
         error_serial = self.get_error_serial(record.context, record.data)
         if error_serial and error_serial in self.error_serials:
-            self.logger.log_info(
-                f"Skipping syncing of `{record.model}` with serial `{error_serial}` due to previous errors.",
-                obj=self.sync,
+            # We want to raise it as exception so it's shown in ingestion issues but can be filtered out.
+            exception = RequiredDependencyFailedSkip(
+                message=f"Skipping syncing of `{record.model}` with serial `{error_serial}` due to previous errors.",
+                model=record.model,
+                context=record.context,
+                data=record.data,
             )
-            return None
+            _, issue = self.create_or_get_sync_issue(
+                exception=exception,
+                ingestion=self.ingestion,
+                model=record.model,
+                context=record.context,
+                data=record.data,
+            )
+            raise exception
         record = self.get_transform_context(record)
         queryset = record.transform_map.target_model.model_class().objects
         model_settings = self.settings.get(record.model, False)
@@ -703,7 +772,7 @@ class IPFabricSyncRunner(object):
 
         self.logger.log_info("Preparing devices", obj=self.sync)
         members = order_members(data.get("virtualchassis", []))
-        devices, virtualchassis = order_devices(data.get("device", []), members)
+        devices, virtualchassis = prepare_devices(data.get("device", []), members)
 
         # We need to store primary IPs of Devices to assign them later
         # since they are not stored on Device object directly
@@ -875,15 +944,69 @@ class IPFabricSyncRunner(object):
 
         for item in items:
             self.sync_item(item, cf, ingestion, stats)
+            self.events_clearer.increment()
 
     @handle_errors
-    def sync_ip_addresses(self, ip_addresses: set[DataRecord]):
+    def sync_devices(
+        self,
+        devices: set[DataRecord],
+        cf: bool = False,
+        ingestion: "IPFabricIngestion" = None,
+    ) -> None:
+        """Sync devices separately to handle resetting primary IP."""
+        if not self.settings.get("device"):
+            self.logger.log_info(
+                "Did not ask to sync devices, skipping.", obj=self.sync
+            )
+            return
+
+        for device in devices:
+            device_obj: "Device | None" = self.sync_item(
+                record=device, cf=cf, ingestion=ingestion
+            )
+
+            if (
+                device_obj is None
+                or device_obj.primary_ip4 is None
+                or device.data.get("loginIp") is not None
+            ):
+                self.events_clearer.increment()
+                continue
+
+            # If device has primary IP assigned in NetBox, but not in IP Fabric, remove it
+            try:
+                connection_name = self.get_db_connection_name()
+                device_obj.refresh_from_db(using=connection_name)
+                device_obj.snapshot()
+                device_obj.primary_ip4 = None
+                device_obj.save(using=connection_name)
+            except Exception as err:
+                _, issue = self.create_or_get_sync_issue(
+                    exception=err,
+                    ingestion=self.ingestion,
+                    message="Error removing primary IP current device.",
+                    model=device.model,
+                    data=device.data,
+                )
+                self.events_clearer.increment()
+                raise IPAddressPrimaryRemovalError(
+                    data=device.data,
+                    model=device.model,
+                    issue_id=issue.pk,
+                ) from err
+            self.events_clearer.increment()
+
+    @handle_errors
+    def sync_ip_addresses(self, ip_addresses: set[DataRecord]) -> None:
         """
         We cannot assign primary IP in signals since IPAddress does not
         contain information whether it is primary or not. And it must be done
         on Device object, so cannot be done via Transform Maps yet since that
         would require another Transform Map for Device.
         So we need to do it manually here.
+
+        Cleaning events queue happens during each cycle to make sure all required
+        operations (primary IP assignment) happen during the same batch.
         """
         if not self.settings.get("ipaddress"):
             self.logger.log_info(
@@ -924,6 +1047,7 @@ class IPFabricSyncRunner(object):
                     model=ip_address.model,
                     data=ip_address.data,
                 )
+                self.events_clearer.increment()
                 raise IPAddressPrimaryRemovalError(
                     data=ip_address.data,
                     model=ip_address.model,
@@ -932,6 +1056,7 @@ class IPFabricSyncRunner(object):
 
             ip_address_obj: "IPAddress | None" = self.sync_item(record=ip_address)
             if ip_address_obj is None or ip_address_obj.assigned_object is None:
+                self.events_clearer.increment()
                 continue
 
             parent_device = ip_address_obj.assigned_object.parent_object
@@ -949,6 +1074,7 @@ class IPFabricSyncRunner(object):
                     self.logger.log_failure(
                         f"Error assigning primary IP to device: {err}", obj=self.sync
                     )
+            self.events_clearer.increment()
 
     def collect_and_sync(self, ingestion=None) -> None:
         self.logger.log_info("Starting data collection.", obj=self.sync)
@@ -967,8 +1093,8 @@ class IPFabricSyncRunner(object):
         self.sync_items(items=records["platform"])
         self.sync_items(items=records["devicerole"])
         self.sync_items(items=records["virtualchassis"])
-        self.sync_items(
-            items=records["device"],
+        self.sync_devices(
+            devices=records["device"],
             cf=self.sync.update_custom_fields,
             ingestion=ingestion,
         )
@@ -982,3 +1108,6 @@ class IPFabricSyncRunner(object):
         self.sync_items(items=records["vrf"])
         self.sync_items(items=records["prefix"])
         self.sync_ip_addresses(ip_addresses=records["ipaddress"])
+
+        # Make sure to clean queue (and memory) at the end
+        self.events_clearer.clear()

{ipfabric_netbox-4.3.2b4 → ipfabric_netbox-4.3.2b6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ipfabric_netbox"
-version = "4.3.2b4"
+version = "4.3.2b6"
 description = "NetBox plugin to sync IP Fabric data into NetBox"
 authors = ["Solution Architecture <solution.architecture@ipfabric.io>"]
 license = "MIT"