ipfabric_netbox 3.1.3__py3-none-any.whl → 3.2.1__py3-none-any.whl

ipfabric_netbox/models.py

@@ -10,6 +10,7 @@ from core.exceptions import SyncError
 from core.models import Job
 from core.signals import handle_deleted_object
 from core.signals import pre_sync
+from dcim.models import Device
 from dcim.models import VirtualChassis
 from dcim.signals import assign_virtualchassis_master
 from django.apps import apps
@@ -21,6 +22,7 @@ from django.db import models
 from django.db import transaction
 from django.db.models import Q
 from django.db.models import signals
+from django.forms.models import model_to_dict
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.module_loading import import_string
@@ -43,6 +45,7 @@ from .choices import IPFabricSnapshotStatusModelChoices
 from .choices import IPFabricSourceTypeChoices
 from .choices import IPFabricSyncTypeChoices
 from .choices import IPFabricTransformMapSourceModelChoices
+from .signals import clear_other_primary_ip
 from .utilities.ipfutils import IPFabric
 from .utilities.ipfutils import IPFabricSyncRunner
 from .utilities.ipfutils import render_jinja2
@@ -71,6 +74,7 @@ IPFabricSupportedSyncModels = Q(
     | Q(app_label="dcim", model="device")
     | Q(app_label="dcim", model="virtualchassis")
     | Q(app_label="dcim", model="interface")
+    | Q(app_label="dcim", model="macaddress")
     | Q(app_label="ipam", model="vlan")
     | Q(app_label="ipam", model="vrf")
     | Q(app_label="ipam", model="prefix")
@@ -138,85 +142,60 @@ class IPFabricTransformMap(NetBoxModel):
         _context["contenttypes"]["ContentType"] = ContentType
         return _context
 
-    def build_relationships(self, uuid, source_data):
+    def build_relationships(self, source_data):
         relationship_maps = self.relationship_maps.all()
         rel_dict = {}
         rel_dict_coalesce = {}
 
         for field in relationship_maps:
-            if field.template:
-                context = {
-                    "object": source_data,
-                }
-                context.update(self.get_models())
-                text = render_jinja2(field.template, context).strip()
-                if text:
-                    try:
-                        pk = int(text)
-                    except ValueError:
-                        pk = text
-
-                    if isinstance(pk, int):
-                        related_object = field.source_model.model_class().objects.get(
-                            pk=pk
-                        )
-                    else:
-                        related_object = ast.literal_eval(pk)
+            if not field.template:
+                continue
+            context = {
+                "object": source_data,
+            }
+            context.update(self.get_models())
+            text = render_jinja2(field.template, context).strip()
+            if text:
+                try:
+                    pk = int(text)
+                except ValueError:
+                    pk = text
+
+                if isinstance(pk, int):
+                    related_object = field.source_model.model_class().objects.get(pk=pk)
+                else:
+                    related_object = ast.literal_eval(pk)
 
-                    if not field.coalesce:
-                        rel_dict[field.target_field] = related_object
+                if not field.coalesce:
+                    rel_dict[field.target_field] = related_object
+                else:
+                    if related_object is None:
+                        # We are searching by this field, so we need to set it to None
+                        rel_dict_coalesce[field.target_field + "__isnull"] = True
                     else:
                         rel_dict_coalesce[field.target_field] = related_object
-            elif uuid and self.relationship_store.get(uuid):
-                object = self.relationship_store[uuid].get(
-                    field.source_model.model_class()
-                )
-                if object:
-                    if not field.coalesce:
-                        rel_dict[field.target_field] = object
-                    else:
-                        rel_dict_coalesce[field.target_field] = object
-
         return rel_dict, rel_dict_coalesce
 
-    def update_or_create_instance(
-        self, data, tags=[], uuid=None, relationship_store={}, logger=None
-    ):
-        self.relationship_store = relationship_store
-        new_data = deepcopy(data)
+    def get_context(self, source_data):
+        new_data = deepcopy(source_data)
         relationship, coalesce_relationship = self.build_relationships(
-            uuid=uuid, source_data=data
+            source_data=source_data
         )
         if relationship:
             new_data["relationship"] = relationship
         if coalesce_relationship:
             new_data["relationship_coalesce"] = coalesce_relationship
         context = self.render(new_data)
-        try:
-            instance, _ = self.target_model.model_class().objects.update_or_create(
-                **context
-            )
-            if instance:
-                apply_tags(instance, tags)
-        except Exception as e:
-            error_message = f"""Failed to create instance:<br/>
-            message: `{e}`<br/>
-            raw data: `{data}`<br/>
-            context: `{context}`<br/>
-            """  # noqa E231 E222
-            logger.log_failure(error_message, obj=self)
-            logger.log_failure(
-                "Ensure that all transform map fields are present.", obj=self
-            )
-            raise SyncError("Unable to update_or_create_instance.")
+        return context
 
+    def update_or_create_instance(self, context, tags=[]):
+        instance, _ = self.target_model.model_class().objects.update_or_create(
+            **context
+        )
+        if instance:
+            apply_tags(instance, tags)
         return instance
 
-    def get_coalesce_fields(self, source_data):
-        data = self.render(source_data)
-        del data["defaults"]
-        return data
-
     def render(self, source_data):
         data = {"defaults": {}}
         for field in self.field_maps.all():
@@ -250,7 +229,10 @@ class IPFabricTransformMap(NetBoxModel):
             if not field.coalesce:
                 data["defaults"][field.target_field] = text
             else:
-                data[field.target_field] = text
+                if text is None:
+                    data[field.target_field + "__isnull"] = True
+                else:
+                    data[field.target_field] = text
 
         if relationship := source_data.get("relationship"):
             data["defaults"].update(relationship)
@@ -597,6 +579,14 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
         # TODO: Add docs url
         return ""
 
+    @property
+    def logger(self):
+        return getattr(self, "_logger", SyncLogging(job=self.pk))
+
+    @logger.setter
+    def logger(self, value):
+        self._logger = value
+
     def get_absolute_url(self):
         return reverse("plugins:ipfabric_netbox:ipfabricsync", args=[self.pk])
 
@@ -711,9 +701,11 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
             )
 
             with checkout(branch):
-                runner.sync_devices(branch=branch)
+                runner.collect_and_sync(branch=branch)
+
+            if self.status != DataSourceStatusChoices.FAILED:
+                self.status = DataSourceStatusChoices.COMPLETED
 
-            self.status = DataSourceStatusChoices.COMPLETED
         except Exception as e:
             self.status = DataSourceStatusChoices.FAILED
             self.logger.log_failure(f"Branch Failed: `{e}`", obj=branch)
@@ -809,6 +801,8 @@ class IPFabricBranch(JobsMixin, Branch):
         statistics = {}
         if job_results:
             for model, stats in job_results["statistics"].items():
+                if not stats["total"]:
+                    continue
                 if stats["total"] > 0:
                     statistics[model] = stats["current"] / stats["total"] * 100
                 else:
@@ -820,7 +814,15 @@ class IPFabricBranch(JobsMixin, Branch):
         with transaction.atomic():
             for change in self.staged_changes.all():
                 logger.debug("Applying change: %s", change)
-                change.apply()
+                try:
+                    change.apply()
+                except Exception as err:
+                    content_type = ContentType.objects.get(pk=change.object_type.pk)
+                    data = model_to_dict(change)["data"]
+                    logger.error(
+                        f"Got error applying change ({content_type}: {data}): {err}"
+                    )
+                    raise
         signals.pre_delete.disconnect(handle_deleted_object)
         self.staged_changes.all().delete()
         signals.pre_delete.connect(handle_deleted_object, sender=StagedChange)
@@ -840,6 +842,7 @@ class IPFabricBranch(JobsMixin, Branch):
         # Begin Sync
         logger.debug(f"Merging {self.name}")
         try:
+            signals.pre_save.connect(clear_other_primary_ip, sender=Device)
             signals.post_save.disconnect(
                 assign_virtualchassis_master, sender=VirtualChassis
             )
@@ -847,6 +850,7 @@ class IPFabricBranch(JobsMixin, Branch):
             signals.post_save.connect(
                 assign_virtualchassis_master, sender=VirtualChassis
             )
+            signals.pre_save.disconnect(clear_other_primary_ip, sender=Device)
             ipfabricsync.status = DataSourceStatusChoices.COMPLETED
         except Exception as e:
             ipfabricsync.status = DataSourceStatusChoices.FAILED

ipfabric_netbox/signals.py

@@ -1,68 +1,29 @@
-from typing import Optional
+import logging
 
-from core.models import ObjectType
 from dcim.models import Device
-from dcim.models import Site
-from extras.choices import CustomFieldTypeChoices
-from extras.choices import CustomFieldUIEditableChoices
-from extras.choices import CustomFieldUIVisibleChoices
-from extras.choices import CustomLinkButtonClassChoices
-from extras.models import CustomField
-from extras.models import CustomLink
-
-from .models import IPFabricBranch
-from .models import IPFabricSource
-
-
-def create_custom_field(
-    field_name: str,
-    label: str,
-    models: list,
-    object_type=None,
-    cf_type: Optional[str] = "type_text",
-    ui_visibility: Optional[str] = "VISIBILITY_READ_ONLY",
-):
-    defaults = {
-        "label": label,
-        "related_object_type": ObjectType.objects.get_for_model(object_type)
-        if object_type
-        else None,
-        "ui_visible": getattr(CustomFieldUIVisibleChoices, "ALWAYS"),
-        "ui_editable": getattr(CustomFieldUIEditableChoices, "NO"),
-    }
-
-    custom_field, _ = CustomField.objects.update_or_create(
-        type=getattr(CustomFieldTypeChoices, cf_type.upper()),
-        name=field_name,
-        defaults=defaults,
-    )
-
-    for model in models:
-        custom_field.object_types.add(ObjectType.objects.get_for_model(model))
-
-
-def ipfabric_netbox_init():
-    create_custom_field(
-        "ipfabric_source",
-        "IP Fabric Source",
-        [Device, Site],
-        cf_type="type_object",
-        object_type=IPFabricSource,
-    )
-    create_custom_field(
-        "ipfabric_branch",
-        "IP Fabric Last Sync",
-        [Device, Site],
-        cf_type="type_object",
-        object_type=IPFabricBranch,
-    )
-    cl, _ = CustomLink.objects.update_or_create(
-        defaults={
-            "link_text": "{% if object.custom_field_data.ipfabric_source is defined %}{% set SOURCE_ID = object.custom_field_data.ipfabric_source %}{% if SOURCE_ID %}IP Fabric{% endif %}{% endif %}",
-            "link_url": '{% if object.custom_field_data.ipfabric_source is defined %}{% set SOURCE_ID = object.custom_field_data.ipfabric_source %}{% if SOURCE_ID %}{% set BASE_URL = object.custom_fields.filter(related_object_type__model="ipfabricsource").first().related_object_type.model_class().objects.get(pk=SOURCE_ID).url %}{{ BASE_URL }}/inventory/devices?options={"filters":{"sn": ["like","{{ object.serial }}"]}}{% endif %}{%endif%}',
-            "new_window": True,
-            "button_class": CustomLinkButtonClassChoices.BLUE,
-        },
-        name="ipfabric",
-    )
-    cl.object_types.add(ObjectType.objects.get_for_model(Device))
+from ipam.models import IPAddress
+
+logger = logging.getLogger("ipfabric_netbox.utilities.ipf_utils")
+
+
+def clear_other_primary_ip(instance: Device, **kwargs) -> None:
+    """
+    When a new device is created with primary IP, make sure there is no other device with the same IP.
+
+    This signal is used when merging stashed changes. It's needed because we cannot
+    guarantee that removing primary IP from Device will happen before adding new one.
+    """
+    try:
+        if not instance.primary_ip:
+            # The device has no primary IP, nothing to do
+            return
+    except IPAddress.DoesNotExist:
+        # THe IP is not created yet, cannot be assigned
+        return
+    try:
+        other_device = Device.objects.get(primary_ip4=instance.primary_ip)
+        if other_device and instance != other_device:
+            other_device.primary_ip4 = None
+            other_device.save()
+    except Device.DoesNotExist:
+        pass

ipfabric_netbox/tables.py

@@ -143,6 +143,8 @@ class SyncTable(NetBoxTable):
 
 
 class StagedChangesTable(NetBoxTable):
+    # There is no view for single StagedChange, remove the link in ID
+    id = tables.Column(verbose_name=_("ID"))
     pk = None
     object_type = tables.Column(
         accessor="object_type.model", verbose_name="Object Type"
@@ -172,6 +174,8 @@ class StagedChangesTable(NetBoxTable):
                 name = f"Tagging object ({record.data['object_id']})"
             elif value == "prefix":
                 name = f"{record.data['prefix']} ({record.data['vrf']})"
+            elif value == "MAC address":
+                name = record.data["mac_address"]
             else:
                 name = record.data
         else: