ipfabric_netbox 4.3.2b1__py3-none-any.whl → 4.3.2b3__py3-none-any.whl

ipfabric_netbox/__init__.py

@@ -6,7 +6,7 @@ class NetboxIPFabricConfig(PluginConfig):
     name = "ipfabric_netbox"
     verbose_name = "NetBox IP Fabric SoT Plugin"
     description = "Sync IP Fabric into NetBox"
-    version = "4.3.2b1"
+    version = "4.3.2b3"
     base_url = "ipfabric"
     min_version = "4.4.0"
 

ipfabric_netbox/data/transform_map.json

@@ -41,8 +41,8 @@
                     "model": "vrf"
                 },
                 "target_field": "vrf",
-                "coalesce": false,
-                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
+                "coalesce": true,
+                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\", \"global\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
             }
         ]
     },
@@ -169,7 +169,7 @@
                 "source_field": "hostname",
                 "target_field": "vc_position",
                 "coalesce": false,
-                "template": "{% if object.virtual_chassis is defined %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}"
+                "template": "{% if object.virtual_chassis %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}"
             }
         ],
         "relationship_maps": [
@@ -180,7 +180,7 @@
                 },
                 "target_field": "virtual_chassis",
                 "coalesce": false,
-                "template": "{% if object.virtual_chassis is defined %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}"
+                "template": "{% if object.virtual_chassis %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}"
             },
             {
                 "source_model": {
@@ -339,7 +339,7 @@
                 "source_field": "primaryIp",
                 "target_field": "mgmt_only",
                 "coalesce": false,
-                "template": "{% if object.primaryIp == object.loginIp %}True{% else %}False{% endif %}"
+                "template": "{% if object.primaryIp and object.primaryIp == object.loginIp %}True{% else %}False{% endif %}"
             },
             {
                 "source_field": "speedValue",
@@ -386,10 +386,10 @@
             "template": "{% if object.mac %}{{ object.mac | mac_to_format(frmt=\"MAC_COLON_TWO\") | upper }}{% else %}{{ \"00:00:00:00:00:01\" | mac_to_format(frmt=\"MAC_COLON_TWO\") | upper }}{% endif %}"
         },
         {
-            "source_field": "id",
+            "source_field": "sn",
             "target_field": "assigned_object_id",
             "coalesce": true,
-            "template": ""
+            "template": "{% if object.nameOriginal %}{{ dcim.Interface.objects.filter(device__serial=object.sn, name=object.nameOriginal).first().pk }}{% else %}{{ dcim.Interface.objects.filter(device__serial=object.sn, name=object.intName).first().pk }}{% endif %}"
         }
         ],
         "relationship_maps": [
@@ -432,7 +432,7 @@
             {
                 "source_field": "name",
                 "target_field": "name",
-                "coalesce": false,
+                "coalesce": true,
                 "template": "{% if object.name is not none %}{{ object.name | string | truncate(64, True) }}{% elif object.dscr is not none %}{{ object.dscr | string | truncate(64, True) }}{% else %}Default Name{% endif %}"
             }
         ],
@@ -443,7 +443,7 @@
                     "model": "device"
                 },
                 "target_field": "device",
-                "coalesce": false,
+                "coalesce": true,
                 "template": "{{ dcim.Device.objects.get(serial=object.deviceSn).pk }}"
             },
             {
@@ -474,7 +474,7 @@
                 "source_field": "vlanName",
                 "target_field": "name",
                 "coalesce": false,
-                "template": "{% if object.vlanName is defined and object.vlanName | lower != \"none\" %}{{ object.vlanName | string | truncate(64, True) }}{% else %}\"\"{% endif %}"
+                "template": "{% if object.vlanName is defined and object.vlanName | lower not in [\"none\", \"\"] %}{{ object.vlanName | string | truncate(64, True) }}{% else %}\"\"{% endif %}"
             },
             {
                 "source_field": "dscr",
@@ -563,7 +563,7 @@
                 },
                 "target_field": "vrf",
                 "coalesce": true,
-                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
+                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\", \"global\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
             },
             {
                 "source_model": {

ipfabric_netbox/exceptions.py

@@ -1,11 +1,5 @@
-from typing import TYPE_CHECKING
-
 from core.exceptions import SyncError
 
-if TYPE_CHECKING:
-    from .models import IPFabricIngestionIssue
-    from .models import IPFabricIngestion
-
 
 class IngestionIssue(Exception):
     """
@@ -47,35 +41,6 @@ class IPAddressDuplicateError(IngestionIssue, SyncError):
         return f"IP address {self.data.get('address')} already exists in {self.model} with coalesce_fields={self.coalesce_fields}."
 
 
-def create_or_get_sync_issue(
-    exception: Exception,
-    ingestion: "IPFabricIngestion",
-    message: str = None,
-    model: str = None,
-    context: dict = None,
-    data: dict = None,
-) -> (bool, "IPFabricIngestionIssue"):
-    """
-    Helper function to handle sync errors and create IPFabricIngestionIssue if needed.
-    """
-    context = context or {}
-
-    # TODO: This is to prevent circular import issues, clean it up later.
-    from .models import IPFabricIngestionIssue
-
-    if not hasattr(exception, "issue_id") or not exception.issue_id:
-        issue = IPFabricIngestionIssue.objects.create(
-            ingestion=ingestion,
-            exception=exception.__class__.__name__,
-            message=message or getattr(exception, "message", str(exception)),
-            model=model,
-            coalesce_fields={k: v for k, v in context.items() if k not in ["defaults"]},
-            defaults=context.get("defaults", dict()),
-            raw_data=data or dict(),
-        )
-        if hasattr(exception, "issue_id"):
-            exception.issue_id = issue.id
-        return True, issue
-    else:
-        issue = IPFabricIngestionIssue.objects.get(id=exception.issue_id)
-        return False, issue
+class IPAddressPrimaryRemovalError(IngestionIssue, SyncError):
+    def __str__(self):
+        return "Error removing primary IP from other device."

ipfabric_netbox/migrations/0021_update_transform_maps.py

@@ -0,0 +1,133 @@
+from django.db import migrations
+
+from ipfabric_netbox.utilities.transform_map import do_change
+from ipfabric_netbox.utilities.transform_map import FieldRecord
+from ipfabric_netbox.utilities.transform_map import RelationshipRecord
+from ipfabric_netbox.utilities.transform_map import TransformMapRecord
+
+
+CHANGES = (
+    TransformMapRecord(
+        source_model="ipaddress",
+        target_model="ipam.ipaddress",
+        relationships=(
+            RelationshipRecord(
+                source_model="ipam.vrf",
+                target_field="vrf",
+                coalesce=True,
+                old_template='{% if object.vrf is defined and object.vrf | string not in ["", "system", "0"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}',
+                new_template='{% if object.vrf is defined and object.vrf | string not in ["", "system", "0", "global"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}',
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="device",
+        target_model="dcim.device",
+        fields=(
+            FieldRecord(
+                source_field="hostname",
+                target_field="vc_position",
+                old_template="{% if object.virtual_chassis is defined %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}",
+                new_template="{% if object.virtual_chassis %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}",
+            ),
+        ),
+        relationships=(
+            RelationshipRecord(
+                source_model="dcim.virtualchassis",
+                target_field="virtual_chassis",
+                old_template="{% if object.virtual_chassis is defined %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}",
+                new_template="{% if object.virtual_chassis %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}",
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="interface",
+        target_model="dcim.interface",
+        fields=(
+            FieldRecord(
+                source_field="primaryIp",
+                target_field="mgmt_only",
+                old_template="{% if object.primaryIp == object.loginIp %}True{% else %}False{% endif %}",
+                new_template="{% if object.primaryIp and object.primaryIp == object.loginIp %}True{% else %}False{% endif %}",
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="interface",
+        target_model="dcim.macaddress",
+        fields=(
+            FieldRecord(
+                source_field="id",
+                new_source_field="sn",
+                target_field="assigned_object_id",
+                old_template="",
+                new_template="{% if object.nameOriginal %}{{ dcim.Interface.objects.filter(device__serial=object.sn, name=object.nameOriginal).first().pk }}{% else %}{{ dcim.Interface.objects.filter(device__serial=object.sn, name=object.intName).first().pk }}{% endif %}",
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="part_number",
+        target_model="dcim.inventoryitem",
+        fields=(
+            FieldRecord(
+                source_field="name",
+                target_field="name",
+                coalesce=True,
+            ),
+        ),
+        relationships=(
+            RelationshipRecord(
+                source_model="dcim.device",
+                target_field="device",
+                coalesce=True,
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="vlan",
+        target_model="ipam.vlan",
+        fields=(
+            FieldRecord(
+                source_field="vlanName",
+                target_field="name",
+                old_template='{% if object.vlanName is defined and object.vlanName | lower != "none" %}{{ object.vlanName | string | truncate(64, True) }}{% else %}""{% endif %}',
+                new_template='{% if object.vlanName is defined and object.vlanName | lower not in ["none", ""] %}{{ object.vlanName | string | truncate(64, True) }}{% else %}""{% endif %}',
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="prefix",
+        target_model="ipam.prefix",
+        relationships=(
+            RelationshipRecord(
+                source_model="ipam.vrf",
+                target_field="vrf",
+                old_template='{% if object.vrf is defined and object.vrf | string not in ["", "system", "0"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}',
+                new_template='{% if object.vrf is defined and object.vrf | string not in ["", "system", "0", "global"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}',
+            ),
+        ),
+    ),
+)
+
+
+def forward_change(apps, schema_editor):
+    """Replace old template with updated version."""
+    do_change(apps, schema_editor, changes=CHANGES, forward=True)
+
+
+def revert_change(apps, schema_editor):
+    """Revert template back to the previous exact template."""
+    do_change(apps, schema_editor, changes=CHANGES, forward=False)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ipfabric_netbox", "0020_clean_scheduled_jobs"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            forward_change,
+            revert_change,
+        ),
+    ]

ipfabric_netbox/models.py

@@ -2,6 +2,7 @@ import ast
 import functools
 import json
 import logging
+import re
 import traceback
 from copy import deepcopy
 from uuid import uuid4
@@ -12,10 +13,8 @@ from core.exceptions import SyncError
 from core.models import Job
 from core.models import ObjectType
 from core.signals import pre_sync
-from dcim.models import Device
+from dcim.models import MACAddress
 from dcim.models import Site
-from dcim.models import VirtualChassis
-from dcim.signals import assign_virtualchassis_master
 from django.apps import apps
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
@@ -50,7 +49,7 @@ from .choices import IPFabricSourceTypeChoices
 from .choices import IPFabricSyncStatusChoices
 from .choices import IPFabricTransformMapSourceModelChoices
 from .choices import required_transform_map_contenttypes
-from .signals import clear_other_primary_ip
+from .signals import assign_primary_mac_address
 from .utilities.ipfutils import IPFabric
 from .utilities.ipfutils import IPFabricSyncRunner
 from .utilities.ipfutils import render_jinja2
@@ -229,6 +228,22 @@ class IPFabricTransformMap(NetBoxModel):
                         rel_dict_coalesce[field.target_field] = related_object
         return rel_dict, rel_dict_coalesce
 
+    def strip_source_data(self, source_data: dict) -> dict:
+        """Strip data according to Transform Map mappings but without rendering templates."""
+        keys = set()
+        for field in self.field_maps.all():
+            keys.add(field.source_field)
+            if field.template:
+                keys.update(
+                    re.findall(r"object\.([a-zA-Z_0-9]+)(?=.*)", field.template)
+                )
+        for field in self.relationship_maps.all():
+            if field.template:
+                keys.update(
+                    re.findall(r"object\.([a-zA-Z_0-9]+)(?=.*)", field.template)
+                )
+        return {k: source_data[k] for k in keys}
+
     def get_context(self, source_data):
         new_data = deepcopy(source_data)
         relationship, coalesce_relationship = self.build_relationships(
@@ -241,7 +256,8 @@ class IPFabricTransformMap(NetBoxModel):
         context = self.render(new_data)
         return context
 
-    def update_or_create_instance(self, context, tags=[], connection_name=None):
+    def update_or_create_instance(self, context, tags=None, connection_name=None):
+        tags = tags or []
         target_class = self.target_model.model_class()
         queryset = target_class.objects.using(connection_name)
 
@@ -272,8 +288,11 @@ class IPFabricTransformMap(NetBoxModel):
                     if field.endswith("__isnull"):
                         context[field[:-8]] = None
                         del context[field]
-                instance = queryset.create(**context, **defaults)
+                # Using queryset.create() creates the object even when it fails on clean()
+                # To to work around it, we do it in two steps to avoid saving it to DB before clean()
+                instance = queryset.model(**context, **defaults)
                 instance.full_clean()
+                instance.save(using=connection_name)
 
             apply_tags(instance, tags, connection_name)
 
@@ -867,9 +886,17 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
             try:
                 active_branch.set(branch)
                 try:
-                    runner.collect_and_sync(
-                        ingestion=IPFabricIngestion.objects.get(pk=ingestion.pk)
-                    )
+                    try:
+                        signals.post_save.connect(
+                            assign_primary_mac_address, sender=MACAddress
+                        )
+                        runner.collect_and_sync(
+                            ingestion=IPFabricIngestion.objects.get(pk=ingestion.pk)
+                        )
+                    finally:
+                        signals.post_save.disconnect(
+                            assign_primary_mac_address, sender=MACAddress
+                        )
                 finally:
                     active_branch.set(None)
             finally:
@@ -1002,15 +1029,7 @@ class IPFabricIngestion(JobsMixin, models.Model):
         # Begin Sync
         logger.debug(f"Merging {self.name}")
         try:
-            signals.pre_save.connect(clear_other_primary_ip, sender=Device)
-            signals.post_save.disconnect(
-                assign_virtualchassis_master, sender=VirtualChassis
-            )
             self.branch.merge(user=self.sync.user)
-            signals.post_save.connect(
-                assign_virtualchassis_master, sender=VirtualChassis
-            )
-            signals.pre_save.disconnect(clear_other_primary_ip, sender=Device)
             ipfabricsync.status = IPFabricSyncStatusChoices.COMPLETED
         except Exception as e:
             ipfabricsync.status = IPFabricSyncStatusChoices.FAILED

ipfabric_netbox/signals.py

@@ -1,39 +1,27 @@
 import logging
 
-from dcim.models import Device
-from ipam.models import IPAddress
+from dcim.models import Interface
+from dcim.models import MACAddress
 from netbox_branching.contextvars import active_branch
 
 logger = logging.getLogger("ipfabric_netbox.utilities.ipf_utils")
 
 
-def clear_other_primary_ip(instance: Device, **kwargs) -> None:
-    """
-    When a new device is created with primary IP, make sure there is no other device with the same IP.
-
-    This signal is used when merging stashed changes. It's needed because we cannot
-    guarantee that removing primary IP from Device will happen before adding new one.
-    """
+def assign_primary_mac_address(instance: MACAddress, **kwargs) -> None:
     try:
-        if not instance.primary_ip:
-            # The device has no primary IP, nothing to do
+        if instance.assigned_object and instance.assigned_object.primary_mac_address:
+            # The Interface already has primary MAC, nothing to do
             return
-    except IPAddress.DoesNotExist:
-        # THe IP is not created yet, cannot be assigned
+    except Interface.DoesNotExist:
+        # The Interface is not created yet, cannot be assigned
         return
-    try:
-        connection_name = None
-        if branch := active_branch.get():
-            connection_name = branch.connection_name
-        other_device = Device.objects.using(connection_name).get(
-            primary_ip4=instance.primary_ip
-        )
-        if other_device and instance != other_device:
-            other_device.snapshot()
-            other_device.primary_ip4 = None
-            other_device.save(using=connection_name)
-    except Device.DoesNotExist:
-        pass
+
+    connection_name = None
+    if branch := active_branch.get():
+        connection_name = branch.connection_name
+    instance.assigned_object.snapshot()
+    instance.assigned_object.primary_mac_address = instance
+    instance.assigned_object.save(using=connection_name)
 
 
 def remove_group_from_syncs(instance, **kwargs):

ipfabric_netbox/templates/ipfabric_netbox/inc/diff.html

@@ -9,7 +9,7 @@
     <div class="row mb-3">
         <div class="col-6">
             <div class="card">
-                <h5 class="card-header">%{ trans "Pre-Change Data" %}</h5>
+                <h5 class="card-header">{% trans "Pre-Change Data" %}</h5>
                 <div class="card-body">
                     {% if prechange_data %}
                     <pre class="change-data">{% for k, v in prechange_data.items %}{% spaceless %}
@@ -17,14 +17,14 @@
                     {% endspaceless %}{% endfor %}
                 </pre>
                     {% else %}
-                    <span class="text-muted">%{ trans "None" %}</span>
+                    <span class="text-muted">{% trans "None" %}</span>
                     {% endif %}
                 </div>
             </div>
         </div>
         <div class="col-6">
             <div class="card">
-                <h5 class="card-header">%{ trans "Post-Change Data" %}</h5>
+                <h5 class="card-header">{% trans "Post-Change Data" %}</h5>
                 <div class="card-body">
                     {% if postchange_data %}
                     <pre class="change-data">{% for k, v in postchange_data.items %}{% spaceless %}
@@ -38,7 +38,7 @@
         <div class="col">
             <div class="card">
                 <div class="card-header">
-                    <h5 class="d-inline">%{ trans "Difference" %}</h5>
+                    <h5 class="d-inline">{% trans "Difference" %}</h5>
                     <!-- <div class="btn-group btn-group-sm float-end noprint">
                             <a {% if prev_change %}href="{% url 'extras:objectchange' pk=prev_change.pk %}"{% else %}disabled{% endif %} class="btn btn-outline-secondary">
                                 <i class="mdi mdi-chevron-left" aria-hidden="true"></i> Previous
@@ -52,11 +52,11 @@
                     {% if diff_added == diff_removed %}
                     <span class="text-muted" style="margin-left: 10px;">
                         {% if object.action == 'create' %}
-                        %{ trans "Object Created" %}
+                        {% trans "Object Created" %}
                         {% elif object.action == 'delete' %}
-                        %{ trans "Object Deleted" %}
+                        {% trans "Object Deleted" %}
                         {% else %}
-                        %{ trans "No Changes" %}
+                        {% trans "No Changes" %}
                         {% endif %}
                     </span>
                     {% else %}

ipfabric_netbox/templates/ipfabric_netbox/ipfabricingestion.html

@@ -91,11 +91,11 @@
       </div>
     </div>
     {% plugin_left_page object %}
-  </div>
-  <div class="col col-md-6">
     <div id="ingestion_statistics" hx-swap-oob="true">
       {% include 'ipfabric_netbox/partials/ingestion_statistics.html' with object=object %}
     </div>
+  </div>
+  <div class="col col-md-6">
     <div id="ingestion_progress" hx-swap-oob="true">
       {% include 'ipfabric_netbox/partials/ingestion_progress.html' with job=object.job %}
     </div>

ipfabric_netbox/tests/test_models.py

@@ -174,7 +174,10 @@ class IPFabricTransformMapModelTestCase(TestCase):
         }
 
         self.site = runner.sync_item(
-            item=site_data, app_label="dcim", model="site", cf=sync.update_custom_fields
+            record=runner.create_new_data_record(
+                app="dcim", model="site", data=site_data
+            ),
+            cf=sync.update_custom_fields,
         )
 
         device_data = {
@@ -210,37 +213,38 @@ class IPFabricTransformMapModelTestCase(TestCase):
             "uptime": 7254180,
             "vendor": "cisco",
             "version": "15.2(4)M1",
+            "virtual_chassis": None,
             "slug": None,
         }
 
         self.mf_obj = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="manufacturer",
+            record=runner.create_new_data_record(
+                app="dcim", model="manufacturer", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.dt_obj = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="devicetype",
+            record=runner.create_new_data_record(
+                app="dcim", model="devicetype", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.platform = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="platform",
+            record=runner.create_new_data_record(
+                app="dcim", model="platform", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.role = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="devicerole",
+            record=runner.create_new_data_record(
+                app="dcim", model="devicerole", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.device_object = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="device",
+            record=runner.create_new_data_record(
+                app="dcim", model="device", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
 
@@ -401,6 +405,7 @@ class IPFabricTransformMapModelTestCase(TestCase):
             "uptime": 7254180,
             "vendor": "cisco",
             "version": "15.2(4)M1",
+            "virtual_chassis": None,
             "slug": None,
         }
 
@@ -415,9 +420,9 @@ class IPFabricTransformMapModelTestCase(TestCase):
         transform_field.template = "{{ object.hostname }} - test"
         transform_field.save()
         device_object = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="device",
+            record=runner.create_new_data_record(
+                app="dcim", model="device", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.assertEqual(device_object.name, "L21PE152 - test")