ipfabric_netbox 3.2.0__py3-none-any.whl → 3.2.2__py3-none-any.whl

ipfabric_netbox/__init__.py

@@ -5,9 +5,9 @@ class NetboxIPFabricConfig(PluginConfig):
     name = "ipfabric_netbox"
     verbose_name = "NetBox IP Fabric SoT Plugin"
     description = "Sync IP Fabric into NetBox"
-    version = "3.2.0"
+    version = "3.2.2"
     base_url = "ipfabric"
-    min_version = "4.2"
+    min_version = "4.2.4"
 
 
 config = NetboxIPFabricConfig

ipfabric_netbox/data/transform_map.json

@@ -43,7 +43,7 @@
                 },
                 "target_field": "vrf",
                 "coalesce": false,
-                "template": "{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}"
+                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
             }
         ]
     },
@@ -82,7 +82,7 @@
                 },
                 "target_field": "manufacturer",
                 "coalesce": false,
-                "template": null
+                "template": "{% set SLUG = object.vendor | slugify %}\n{{ dcim.Manufacturer.objects.get(slug=SLUG).pk }}"
             }
         ]
     },
@@ -194,7 +194,7 @@
                 },
                 "target_field": "platform",
                 "coalesce": false,
-                "template": null
+                "template": "{% set SLUG = object.vendor | slugify %}{% if object.family %}{% set SLUG = SLUG ~ \"_\" ~ object.family | slugify %}{% endif %}\n{{ dcim.Platform.objects.get(slug=SLUG).pk }}"
             },
             {
                 "source_model": {
@@ -203,7 +203,7 @@
                 },
                 "target_field": "site",
                 "coalesce": false,
-                "template": null
+                "template": "{% if object.siteName is defined %}{% set SLUG = object.siteName | slugify %}\n{{ dcim.Site.objects.get(slug=SLUG).pk }}{% else %}None{% endif %}"
             },
             {
                 "source_model": {
@@ -212,7 +212,7 @@
                 },
                 "target_field": "device_type",
                 "coalesce": false,
-                "template": null
+                "template": "{% if object.model != none %}{% set SLUG = object.model | string | slugify %}{% else %}{% set SLUG = object.vendor | slugify ~ \"-\" ~ object.family | slugify ~ \"-\" ~ object.platform %}{% endif %}{{ dcim.DeviceType.objects.get(slug=SLUG).pk }}"
             },
             {
                 "source_model": {
@@ -221,7 +221,7 @@
                 },
                 "target_field": "role",
                 "coalesce": false,
-                "template": null
+                "template": "{% if object.devType is defined %}{% set SLUG = object.devType | slugify %}{{ dcim.DeviceRole.objects.get(slug=SLUG).pk }}{% endif %}"
             }
         ]
     },
@@ -295,7 +295,7 @@
                 },
                 "target_field": "manufacturer",
                 "coalesce": false,
-                "template": null
+                "template": "{% set SLUG = object.vendor | slugify %}\n{{ dcim.Manufacturer.objects.get(slug=SLUG).pk }}"
             }
         ]
     },
@@ -370,7 +370,7 @@
                 },
                 "target_field": "device",
                 "coalesce": true,
-                "template": null
+                "template": "{{ dcim.Device.objects.get(serial=object.sn).pk }}"
             }
         ]
     },
@@ -431,7 +431,7 @@
                 "source_field": "pid",
                 "target_field": "part_id",
                 "coalesce": false,
-                "template": ""
+                "template": "{% if object.pid and object.pid != \"None\" %}{{ object.pid }}{% else %}unknown{% endif %}"
             },
             {
                 "source_field": "sn",
@@ -454,7 +454,7 @@
                 },
                 "target_field": "device",
                 "coalesce": false,
-                "template": "{{ dcim.Device.objects.filter(serial=object.deviceSn).first().pk }}"
+                "template": "{{ dcim.Device.objects.get(serial=object.deviceSn).pk }}"
             },
             {
                 "source_model": {
@@ -463,7 +463,7 @@
                 },
                 "target_field": "manufacturer",
                 "coalesce": false,
-                "template": null
+                "template": "{% if object.vendor is defined %}{% set SLUG = object.vendor | slugify %}\n{{ dcim.Manufacturer.objects.get(slug=SLUG).pk }}{% endif %}"
             }
         ]
     },
@@ -485,7 +485,7 @@
                 "source_field": "vlanName",
                 "target_field": "name",
                 "coalesce": false,
-                "template": "{{ object.vlanName or \"\"}}"
+                "template": "{% if object.vlanName is defined and object.vlanName | lower != \"none\" %}{{ object.vlanName | string | truncate(64, True) }}{% else %}\"\"{% endif %}"
             },
             {
                 "source_field": "dscr",
@@ -508,7 +508,7 @@
                 },
                 "target_field": "site",
                 "coalesce": true,
-                "template": null
+                "template": "{% if object.siteName is defined %}{% set SLUG = object.siteName | slugify %}\n{{ dcim.Site.objects.get(slug=SLUG).pk }}{% else %}None{% endif %}"
             }
         ]
     },
@@ -565,7 +565,7 @@
                 "source_field": "siteName",
                 "target_field": "scope_id",
                 "coalesce": true,
-                "template": "{% if object.siteName is defined %}{% set SLUG = object.siteName | slugify %}\n{{ dcim.Site.objects.filter(slug=SLUG).first().pk }}{% else %}None{% endif %}"
+                "template": "{% if object.siteName is defined %}{% set SLUG = object.siteName | slugify %}\n{{ dcim.Site.objects.get(slug=SLUG).pk }}{% else %}None{% endif %}"
             }
         ],
         "relationship_maps": [
@@ -576,7 +576,7 @@
                 },
                 "target_field": "vrf",
                 "coalesce": true,
-                "template": "{% if object.vrf %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
+                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
             },
             {
                 "source_model": {
@@ -618,7 +618,7 @@
                 },
                 "target_field": "master",
                 "coalesce": false,
-                "template": "{{ dcim.Device.objects.filter(serial=object.sn).first().pk }}"
+                "template": "{% set DEVICE = dcim.Device.objects.filter(serial=object.sn).first() %}{% if DEVICE %}{{ DEVICE.pk }}{% endif %}"
             }
         ]
     }

ipfabric_netbox/exceptions.py

@@ -0,0 +1,24 @@
+from core.exceptions import SyncError
+
+
+class ErrorMixin(Exception):
+    model: str = ""
+    defaults: dict[str, str] = {}
+    coalesce_fields: dict[str, str] = {}
+
+    def __init__(self, model: str, context: dict, data: dict = None):
+        super().__init__()
+        self.model = model
+        self.data = data or {}
+        self.defaults = context.pop("defaults", {})
+        self.coalesce_fields = context
+
+
+class SearchError(ErrorMixin, LookupError):
+    def __str__(self):
+        return f"{self.model} with these keys not found: {self.coalesce_fields}."
+
+
+class SyncDataError(ErrorMixin, SyncError):
+    def __str__(self):
+        return f"Sync failed for {self.model}: coalesce_fields={self.coalesce_fields} defaults={self.defaults}."

ipfabric_netbox/forms.py

@@ -1,12 +1,12 @@
 import copy
 
 from core.choices import DataSourceStatusChoices
+from core.choices import JobIntervalChoices
 from django import forms
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
-from extras.choices import DurationChoices
 from netbox.forms import NetBoxModelFilterSetForm
 from netbox.forms import NetBoxModelForm
 from netbox.forms.mixins import SavedFiltersMixin
@@ -476,7 +476,7 @@ class IPFabricSyncForm(NetBoxModelForm):
         required=False,
         min_value=1,
         label=_("Recurs every"),
-        widget=NumberWithOptions(options=DurationChoices),
+        widget=NumberWithOptions(options=JobIntervalChoices),
         help_text=_("Interval at which this sync is re-run (in minutes)"),
     )
     auto_merge = forms.BooleanField(

ipfabric_netbox/migrations/0009_transformmap_changes_for_netbox_v4_2.py

@@ -41,9 +41,6 @@ def get_current_map_field(apps: "apps", target_model: str) -> "IPFabricTransform
     )
 
 
-# TODO: Add MAC Address to existing IPFabricSync instances if Interface is allowed
-
-
 def create_transform_map(apps: "apps", schema_editor: "BaseDatabaseSchemaEditor"):
     """Create transform map structure for MAC Address (new in NetBox v4.2).
     Keeps current template for MAC Address if it exists.

ipfabric_netbox/migrations/0010_remove_uuid_from_get_or_create.py

@@ -0,0 +1,95 @@
+from typing import TYPE_CHECKING
+
+from django.db import migrations
+
+from ipfabric_netbox.utilities.transform_map import get_transform_map
+
+if TYPE_CHECKING:
+    from django.apps import apps
+    from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+
+
+items_to_change = {
+    ("ip_address", "ipam", "ipaddress"): {"fields": [], "relationships": ["vrf"]},
+    ("device", "dcim", "platform"): {"fields": [], "relationships": ["manufacturer"]},
+    ("device", "dcim", "device"): {
+        "fields": [],
+        "relationships": ["platform", "site", "device_type", "role"],
+    },
+    ("device", "dcim", "devicetype"): {"fields": [], "relationships": ["manufacturer"]},
+    ("interface", "dcim", "interface"): {"fields": [], "relationships": ["device"]},
+    ("part_number", "dcim", "inventoryitem"): {
+        "fields": ["part_id"],
+        "relationships": ["device", "manufacturer"],
+    },
+    ("vlan", "ipam", "vlan"): {"fields": ["name"], "relationships": ["site"]},
+    ("prefix", "ipam", "prefix"): {"fields": ["scope_id"], "relationships": ["vrf"]},
+    ("virtualchassis", "dcim", "virtualchassis"): {
+        "fields": [],
+        "relationships": ["master"],
+    },
+}
+
+
+def add_templates(apps: "apps", schema_editor: "BaseDatabaseSchemaEditor"):
+    transform_map_data = get_transform_map()
+
+    for item in transform_map_data:
+        transform_map_id = (
+            item["data"]["source_model"],
+            item["data"]["target_model"]["app_label"],
+            item["data"]["target_model"]["model"],
+        )
+        if transform_map_id not in items_to_change:
+            continue
+        source_model, app_label, target_model = transform_map_id
+        transform_map = apps.get_model(
+            "ipfabric_netbox", "IPFabricTransformMap"
+        ).objects.get(
+            source_model=source_model,
+            target_model=apps.get_model("contenttypes", "ContentType").objects.get(
+                app_label=app_label,
+                model=target_model,
+            ),
+        )
+
+        change_data = items_to_change[transform_map_id]
+
+        for field_map in item["field_maps"]:
+            if field_map["target_field"] not in change_data["fields"]:
+                continue
+            field = apps.get_model(
+                "ipfabric_netbox", "IPFabricTransformField"
+            ).objects.get(
+                target_field=field_map["target_field"], transform_map=transform_map
+            )
+            field.template = field_map["template"]
+            field.save()
+
+        for relationship_map in item["relationship_maps"]:
+            if relationship_map["target_field"] not in change_data["relationships"]:
+                continue
+            relationship = apps.get_model(
+                "ipfabric_netbox", "IPFabricRelationshipField"
+            ).objects.get(
+                target_field=relationship_map["target_field"],
+                transform_map=transform_map,
+            )
+            relationship.template = relationship_map["template"]
+            relationship.save()
+
+
+def return_templates(apps: "apps", schema_editor: "BaseDatabaseSchemaEditor"):
+    """It would be way too complex to revert this migration and there is no need to do so.
+    The original code works the same with or without the templates."""
+    pass
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ipfabric_netbox", "0009_transformmap_changes_for_netbox_v4_2"),
+    ]
+
+    operations = [
+        migrations.RunPython(add_templates, return_templates),
+    ]

ipfabric_netbox/models.py

@@ -10,6 +10,7 @@ from core.exceptions import SyncError
 from core.models import Job
 from core.signals import handle_deleted_object
 from core.signals import pre_sync
+from dcim.models import Device
 from dcim.models import VirtualChassis
 from dcim.signals import assign_virtualchassis_master
 from django.apps import apps
@@ -21,6 +22,7 @@ from django.db import models
 from django.db import transaction
 from django.db.models import Q
 from django.db.models import signals
+from django.forms.models import model_to_dict
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.module_loading import import_string
@@ -43,6 +45,7 @@ from .choices import IPFabricSnapshotStatusModelChoices
 from .choices import IPFabricSourceTypeChoices
 from .choices import IPFabricSyncTypeChoices
 from .choices import IPFabricTransformMapSourceModelChoices
+from .signals import clear_other_primary_ip
 from .utilities.ipfutils import IPFabric
 from .utilities.ipfutils import IPFabricSyncRunner
 from .utilities.ipfutils import render_jinja2
@@ -139,85 +142,60 @@ class IPFabricTransformMap(NetBoxModel):
         _context["contenttypes"]["ContentType"] = ContentType
         return _context
 
-    def build_relationships(self, uuid, source_data):
+    def build_relationships(self, source_data):
         relationship_maps = self.relationship_maps.all()
         rel_dict = {}
         rel_dict_coalesce = {}
 
         for field in relationship_maps:
-            if field.template:
-                context = {
-                    "object": source_data,
-                }
-                context.update(self.get_models())
-                text = render_jinja2(field.template, context).strip()
-                if text:
-                    try:
-                        pk = int(text)
-                    except ValueError:
-                        pk = text
-
-                    if isinstance(pk, int):
-                        related_object = field.source_model.model_class().objects.get(
-                            pk=pk
-                        )
-                    else:
-                        related_object = ast.literal_eval(pk)
+            if not field.template:
+                continue
+            context = {
+                "object": source_data,
+            }
+            context.update(self.get_models())
+            text = render_jinja2(field.template, context).strip()
+            if text:
+                try:
+                    pk = int(text)
+                except ValueError:
+                    pk = text
+
+                if isinstance(pk, int):
+                    related_object = field.source_model.model_class().objects.get(pk=pk)
+                else:
+                    related_object = ast.literal_eval(pk)
 
-                    if not field.coalesce:
-                        rel_dict[field.target_field] = related_object
+                if not field.coalesce:
+                    rel_dict[field.target_field] = related_object
+                else:
+                    if related_object is None:
+                        # We are searching by this field, so we need to set it to None
+                        rel_dict_coalesce[field.target_field + "__isnull"] = True
                     else:
                         rel_dict_coalesce[field.target_field] = related_object
-            elif uuid and self.relationship_store.get(uuid):
-                object = self.relationship_store[uuid].get(
-                    field.source_model.model_class()
-                )
-                if object:
-                    if not field.coalesce:
-                        rel_dict[field.target_field] = object
-                    else:
-                        rel_dict_coalesce[field.target_field] = object
-
         return rel_dict, rel_dict_coalesce
 
-    def update_or_create_instance(
-        self, data, tags=[], uuid=None, relationship_store={}, logger=None
-    ):
-        self.relationship_store = relationship_store
-        new_data = deepcopy(data)
+    def get_context(self, source_data):
+        new_data = deepcopy(source_data)
         relationship, coalesce_relationship = self.build_relationships(
-            uuid=uuid, source_data=data
+            source_data=source_data
         )
         if relationship:
             new_data["relationship"] = relationship
         if coalesce_relationship:
             new_data["relationship_coalesce"] = coalesce_relationship
         context = self.render(new_data)
-        try:
-            instance, _ = self.target_model.model_class().objects.update_or_create(
-                **context
-            )
-            if instance:
-                apply_tags(instance, tags)
-        except Exception as e:
-            error_message = f"""Failed to create instance of `{str(self.target_model)}`:<br/>
-            message: `{e}`<br/>
-            raw data: `{data}`<br/>
-            context: `{context}`<br/>
-            """  # noqa E231 E222
-            logger.log_failure(error_message, obj=self)
-            logger.log_failure(
-                "Ensure that all transform map fields are present.", obj=self
-            )
-            raise SyncError("Unable to update_or_create_instance.")
+        return context
 
+    def update_or_create_instance(self, context, tags=[]):
+        instance, _ = self.target_model.model_class().objects.update_or_create(
+            **context
+        )
+        if instance:
+            apply_tags(instance, tags)
         return instance
 
-    def get_coalesce_fields(self, source_data):
-        data = self.render(source_data)
-        del data["defaults"]
-        return data
-
     def render(self, source_data):
         data = {"defaults": {}}
         for field in self.field_maps.all():
@@ -251,7 +229,10 @@ class IPFabricTransformMap(NetBoxModel):
             if not field.coalesce:
                 data["defaults"][field.target_field] = text
             else:
-                data[field.target_field] = text
+                if text is None:
+                    data[field.target_field + "__isnull"] = True
+                else:
+                    data[field.target_field] = text
 
         if relationship := source_data.get("relationship"):
             data["defaults"].update(relationship)
@@ -720,9 +701,11 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
             )
 
             with checkout(branch):
-                runner.sync_devices(branch=branch)
+                runner.collect_and_sync(branch=branch)
+
+            if self.status != DataSourceStatusChoices.FAILED:
+                self.status = DataSourceStatusChoices.COMPLETED
 
-            self.status = DataSourceStatusChoices.COMPLETED
         except Exception as e:
             self.status = DataSourceStatusChoices.FAILED
             self.logger.log_failure(f"Branch Failed: `{e}`", obj=branch)
@@ -818,6 +801,8 @@ class IPFabricBranch(JobsMixin, Branch):
         statistics = {}
         if job_results:
             for model, stats in job_results["statistics"].items():
+                if not stats["total"]:
+                    continue
                 if stats["total"] > 0:
                     statistics[model] = stats["current"] / stats["total"] * 100
                 else:
@@ -829,7 +814,15 @@ class IPFabricBranch(JobsMixin, Branch):
         with transaction.atomic():
             for change in self.staged_changes.all():
                 logger.debug("Applying change: %s", change)
-                change.apply()
+                try:
+                    change.apply()
+                except Exception as err:
+                    content_type = ContentType.objects.get(pk=change.object_type.pk)
+                    data = model_to_dict(change)["data"]
+                    logger.error(
+                        f"Got error applying change ({content_type}: {data}): {err}"
+                    )
+                    raise
         signals.pre_delete.disconnect(handle_deleted_object)
         self.staged_changes.all().delete()
         signals.pre_delete.connect(handle_deleted_object, sender=StagedChange)
@@ -849,6 +842,7 @@ class IPFabricBranch(JobsMixin, Branch):
         # Begin Sync
         logger.debug(f"Merging {self.name}")
         try:
+            signals.pre_save.connect(clear_other_primary_ip, sender=Device)
             signals.post_save.disconnect(
                 assign_virtualchassis_master, sender=VirtualChassis
             )
@@ -856,6 +850,7 @@ class IPFabricBranch(JobsMixin, Branch):
             signals.post_save.connect(
                 assign_virtualchassis_master, sender=VirtualChassis
             )
+            signals.pre_save.disconnect(clear_other_primary_ip, sender=Device)
             ipfabricsync.status = DataSourceStatusChoices.COMPLETED
         except Exception as e:
             ipfabricsync.status = DataSourceStatusChoices.FAILED

ipfabric_netbox/signals.py

@@ -0,0 +1,29 @@
+import logging
+
+from dcim.models import Device
+from ipam.models import IPAddress
+
+logger = logging.getLogger("ipfabric_netbox.utilities.ipf_utils")
+
+
+def clear_other_primary_ip(instance: Device, **kwargs) -> None:
+    """
+    When a new device is created with primary IP, make sure there is no other device with the same IP.
+
+    This signal is used when merging stashed changes. It's needed because we cannot
+    guarantee that removing primary IP from Device will happen before adding new one.
+    """
+    try:
+        if not instance.primary_ip:
+            # The device has no primary IP, nothing to do
+            return
+    except IPAddress.DoesNotExist:
+        # THe IP is not created yet, cannot be assigned
+        return
+    try:
+        other_device = Device.objects.get(primary_ip4=instance.primary_ip)
+        if other_device and instance != other_device:
+            other_device.primary_ip4 = None
+            other_device.save()
+    except Device.DoesNotExist:
+        pass

ipfabric_netbox/tests/test_models.py

@@ -1,6 +1,5 @@
 import copy
 import json
-import uuid
 
 from dcim.models import Device
 from django.contrib.contenttypes.models import ContentType
@@ -155,10 +154,9 @@ class IPFabricTransformMapModelTestCase(TestCase):
             },
             sync=sync,
         )
-        device_uuid = str(uuid.uuid4())
 
         site_data = {
-            "siteName": "Site 1",
+            "siteName": "MPLS",
             "devicesCount": 1,
             "usersCount": 2,
             "stpDCount": 0,
@@ -169,9 +167,7 @@ class IPFabricTransformMapModelTestCase(TestCase):
             "networksCount": 6,
         }
 
-        self.site = runner.get_model_or_update(
-            "dcim", "site", site_data, uuid=device_uuid
-        )
+        self.site = runner.get_model_or_update("dcim", "site", site_data)
 
         device_data = {
             "id": "961251111",
@@ -209,24 +205,14 @@ class IPFabricTransformMapModelTestCase(TestCase):
             "slug": None,
         }
 
-        self.mf_obj = runner.get_model_or_update(
-            "dcim", "manufacturer", device_data, uuid=device_uuid
-        )
-        self.dt_obj = runner.get_model_or_update(
-            "dcim", "devicetype", device_data, uuid=device_uuid
-        )
+        self.mf_obj = runner.get_model_or_update("dcim", "manufacturer", device_data)
+        self.dt_obj = runner.get_model_or_update("dcim", "devicetype", device_data)
 
-        self.platform = runner.get_model_or_update(
-            "dcim", "platform", device_data, uuid=device_uuid
-        )
+        self.platform = runner.get_model_or_update("dcim", "platform", device_data)
 
-        self.role = runner.get_model_or_update(
-            "dcim", "devicerole", device_data, uuid=device_uuid
-        )
+        self.role = runner.get_model_or_update("dcim", "devicerole", device_data)
 
-        self.device_object = runner.get_model_or_update(
-            "dcim", "device", device_data, uuid=device_uuid
-        )
+        self.device_object = runner.get_model_or_update("dcim", "device", device_data)
 
     def test_transform_map(self):
         site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
@@ -318,7 +304,8 @@ class IPFabricTransformMapModelTestCase(TestCase):
             "routersCount": 0,
             "networksCount": 6,
         }
-        object = site_transform_map.update_or_create_instance(data=data)
+        context = site_transform_map.get_context(data)
+        object = site_transform_map.update_or_create_instance(context)
         self.assertEqual(object.name, "Site 1")
         self.assertEqual(object.slug, "site-1")
 
@@ -353,7 +340,6 @@ class IPFabricTransformMapModelTestCase(TestCase):
             },
             sync=sync,
         )
-        device_uuid = str(uuid.uuid4())
 
         device_data = {
             "id": "961251111",
@@ -401,7 +387,5 @@ class IPFabricTransformMapModelTestCase(TestCase):
         )
         transform_field.template = "{{ object.hostname }} - test"
         transform_field.save()
-        device_object = runner.get_model_or_update(
-            "dcim", "device", device_data, uuid=device_uuid
-        )
+        device_object = runner.get_model_or_update("dcim", "device", device_data)
         self.assertEqual(device_object.name, "L21PE152 - test")