ipfabric_netbox 4.3.2b1__tar.gz → 4.3.2b3__tar.gz

{ipfabric_netbox-4.3.2b1 → ipfabric_netbox-4.3.2b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipfabric_netbox
-Version: 4.3.2b1
+Version: 4.3.2b3
 Summary: NetBox plugin to sync IP Fabric data into NetBox
 License: MIT
 Keywords: netbox,ipfabric,plugin,sync
@@ -19,6 +19,7 @@ Provides-Extra: ipfabric-6-10
 Provides-Extra: ipfabric-7-0
 Provides-Extra: ipfabric-7-2
 Provides-Extra: ipfabric-7-3
+Requires-Dist: httpx (>0.26,<0.29)
 Requires-Dist: ipfabric (>=6.10.0,<6.11.0) ; extra == "ipfabric_6_10" and extra != "ipfabric_7_0" and extra != "ipfabric_7_2" and extra != "ipfabric_7_3"
 Requires-Dist: ipfabric (>=6.6.4) ; extra != "ipfabric_6_10" and extra != "ipfabric_7_0" and extra != "ipfabric_7_2" and extra != "ipfabric_7_3"
 Requires-Dist: ipfabric (>=7.0.0,<7.1.0) ; extra != "ipfabric_6_10" and extra == "ipfabric_7_0" and extra != "ipfabric_7_2" and extra != "ipfabric_7_3"

{ipfabric_netbox-4.3.2b1 → ipfabric_netbox-4.3.2b3}/ipfabric_netbox/__init__.py

@@ -6,7 +6,7 @@ class NetboxIPFabricConfig(PluginConfig):
     name = "ipfabric_netbox"
     verbose_name = "NetBox IP Fabric SoT Plugin"
     description = "Sync IP Fabric into NetBox"
-    version = "4.3.2b1"
+    version = "4.3.2b3"
     base_url = "ipfabric"
     min_version = "4.4.0"
 

{ipfabric_netbox-4.3.2b1 → ipfabric_netbox-4.3.2b3}/ipfabric_netbox/data/transform_map.json

@@ -41,8 +41,8 @@
                     "model": "vrf"
                 },
                 "target_field": "vrf",
-                "coalesce": false,
-                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
+                "coalesce": true,
+                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\", \"global\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
             }
         ]
     },
@@ -169,7 +169,7 @@
                 "source_field": "hostname",
                 "target_field": "vc_position",
                 "coalesce": false,
-                "template": "{% if object.virtual_chassis is defined %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}"
+                "template": "{% if object.virtual_chassis %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}"
             }
         ],
         "relationship_maps": [
@@ -180,7 +180,7 @@
                 },
                 "target_field": "virtual_chassis",
                 "coalesce": false,
-                "template": "{% if object.virtual_chassis is defined %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}"
+                "template": "{% if object.virtual_chassis %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}"
             },
             {
                 "source_model": {
@@ -339,7 +339,7 @@
                 "source_field": "primaryIp",
                 "target_field": "mgmt_only",
                 "coalesce": false,
-                "template": "{% if object.primaryIp == object.loginIp %}True{% else %}False{% endif %}"
+                "template": "{% if object.primaryIp and object.primaryIp == object.loginIp %}True{% else %}False{% endif %}"
             },
             {
                 "source_field": "speedValue",
@@ -386,10 +386,10 @@
             "template": "{% if object.mac %}{{ object.mac | mac_to_format(frmt=\"MAC_COLON_TWO\") | upper }}{% else %}{{ \"00:00:00:00:00:01\" | mac_to_format(frmt=\"MAC_COLON_TWO\") | upper }}{% endif %}"
         },
         {
-            "source_field": "id",
+            "source_field": "sn",
             "target_field": "assigned_object_id",
             "coalesce": true,
-            "template": ""
+            "template": "{% if object.nameOriginal %}{{ dcim.Interface.objects.filter(device__serial=object.sn, name=object.nameOriginal).first().pk }}{% else %}{{ dcim.Interface.objects.filter(device__serial=object.sn, name=object.intName).first().pk }}{% endif %}"
         }
         ],
         "relationship_maps": [
@@ -432,7 +432,7 @@
             {
                 "source_field": "name",
                 "target_field": "name",
-                "coalesce": false,
+                "coalesce": true,
                 "template": "{% if object.name is not none %}{{ object.name | string | truncate(64, True) }}{% elif object.dscr is not none %}{{ object.dscr | string | truncate(64, True) }}{% else %}Default Name{% endif %}"
             }
         ],
@@ -443,7 +443,7 @@
                     "model": "device"
                 },
                 "target_field": "device",
-                "coalesce": false,
+                "coalesce": true,
                 "template": "{{ dcim.Device.objects.get(serial=object.deviceSn).pk }}"
             },
             {
@@ -474,7 +474,7 @@
                 "source_field": "vlanName",
                 "target_field": "name",
                 "coalesce": false,
-                "template": "{% if object.vlanName is defined and object.vlanName | lower != \"none\" %}{{ object.vlanName | string | truncate(64, True) }}{% else %}\"\"{% endif %}"
+                "template": "{% if object.vlanName is defined and object.vlanName | lower not in [\"none\", \"\"] %}{{ object.vlanName | string | truncate(64, True) }}{% else %}\"\"{% endif %}"
             },
             {
                 "source_field": "dscr",
@@ -563,7 +563,7 @@
                 },
                 "target_field": "vrf",
                 "coalesce": true,
-                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
+                "template": "{% if object.vrf is defined and object.vrf | string not in [\"\", \"system\", \"0\", \"global\"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}"
             },
             {
                 "source_model": {

ipfabric_netbox-4.3.2b3/ipfabric_netbox/exceptions.py

@@ -0,0 +1,46 @@
+from core.exceptions import SyncError
+
+
+class IngestionIssue(Exception):
+    """
+    This exception is used to indicate an issue during the ingestion process.
+    """
+
+    # Store created issue object ID if it exists for this exception
+    issue_id = None
+    model: str = ""
+    defaults: dict[str, str] = {}
+    coalesce_fields: dict[str, str] = {}
+
+    def __init__(self, model: str, data: dict, context: dict = None, issue_id=None):
+        super().__init__()
+        self.model = model
+        self.data = data
+        context = context or {}
+        self.defaults = context.pop("defaults", {})
+        self.coalesce_fields = context
+        self.issue_id = issue_id
+
+
+class SearchError(IngestionIssue, LookupError):
+    def __init__(self, message: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class SyncDataError(IngestionIssue, SyncError):
+    def __str__(self):
+        return f"Sync failed for {self.model}: coalesce_fields={self.coalesce_fields} defaults={self.defaults}."
+
+
+class IPAddressDuplicateError(IngestionIssue, SyncError):
+    def __str__(self):
+        return f"IP address {self.data.get('address')} already exists in {self.model} with coalesce_fields={self.coalesce_fields}."
+
+
+class IPAddressPrimaryRemovalError(IngestionIssue, SyncError):
+    def __str__(self):
+        return "Error removing primary IP from other device."

ipfabric_netbox-4.3.2b3/ipfabric_netbox/migrations/0021_update_transform_maps.py

@@ -0,0 +1,133 @@
+from django.db import migrations
+
+from ipfabric_netbox.utilities.transform_map import do_change
+from ipfabric_netbox.utilities.transform_map import FieldRecord
+from ipfabric_netbox.utilities.transform_map import RelationshipRecord
+from ipfabric_netbox.utilities.transform_map import TransformMapRecord
+
+
+CHANGES = (
+    TransformMapRecord(
+        source_model="ipaddress",
+        target_model="ipam.ipaddress",
+        relationships=(
+            RelationshipRecord(
+                source_model="ipam.vrf",
+                target_field="vrf",
+                coalesce=True,
+                old_template='{% if object.vrf is defined and object.vrf | string not in ["", "system", "0"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}',
+                new_template='{% if object.vrf is defined and object.vrf | string not in ["", "system", "0", "global"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}',
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="device",
+        target_model="dcim.device",
+        fields=(
+            FieldRecord(
+                source_field="hostname",
+                target_field="vc_position",
+                old_template="{% if object.virtual_chassis is defined %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}",
+                new_template="{% if object.virtual_chassis %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}",
+            ),
+        ),
+        relationships=(
+            RelationshipRecord(
+                source_model="dcim.virtualchassis",
+                target_field="virtual_chassis",
+                old_template="{% if object.virtual_chassis is defined %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}",
+                new_template="{% if object.virtual_chassis %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}",
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="interface",
+        target_model="dcim.interface",
+        fields=(
+            FieldRecord(
+                source_field="primaryIp",
+                target_field="mgmt_only",
+                old_template="{% if object.primaryIp == object.loginIp %}True{% else %}False{% endif %}",
+                new_template="{% if object.primaryIp and object.primaryIp == object.loginIp %}True{% else %}False{% endif %}",
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="interface",
+        target_model="dcim.macaddress",
+        fields=(
+            FieldRecord(
+                source_field="id",
+                new_source_field="sn",
+                target_field="assigned_object_id",
+                old_template="",
+                new_template="{% if object.nameOriginal %}{{ dcim.Interface.objects.filter(device__serial=object.sn, name=object.nameOriginal).first().pk }}{% else %}{{ dcim.Interface.objects.filter(device__serial=object.sn, name=object.intName).first().pk }}{% endif %}",
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="part_number",
+        target_model="dcim.inventoryitem",
+        fields=(
+            FieldRecord(
+                source_field="name",
+                target_field="name",
+                coalesce=True,
+            ),
+        ),
+        relationships=(
+            RelationshipRecord(
+                source_model="dcim.device",
+                target_field="device",
+                coalesce=True,
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="vlan",
+        target_model="ipam.vlan",
+        fields=(
+            FieldRecord(
+                source_field="vlanName",
+                target_field="name",
+                old_template='{% if object.vlanName is defined and object.vlanName | lower != "none" %}{{ object.vlanName | string | truncate(64, True) }}{% else %}""{% endif %}',
+                new_template='{% if object.vlanName is defined and object.vlanName | lower not in ["none", ""] %}{{ object.vlanName | string | truncate(64, True) }}{% else %}""{% endif %}',
+            ),
+        ),
+    ),
+    TransformMapRecord(
+        source_model="prefix",
+        target_model="ipam.prefix",
+        relationships=(
+            RelationshipRecord(
+                source_model="ipam.vrf",
+                target_field="vrf",
+                old_template='{% if object.vrf is defined and object.vrf | string not in ["", "system", "0"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}',
+                new_template='{% if object.vrf is defined and object.vrf | string not in ["", "system", "0", "global"] %}{{ ipam.VRF.objects.filter(name=object.vrf).first().pk }}{% else %}None{% endif %}',
+            ),
+        ),
+    ),
+)
+
+
+def forward_change(apps, schema_editor):
+    """Replace old template with updated version."""
+    do_change(apps, schema_editor, changes=CHANGES, forward=True)
+
+
+def revert_change(apps, schema_editor):
+    """Revert template back to the previous exact template."""
+    do_change(apps, schema_editor, changes=CHANGES, forward=False)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ipfabric_netbox", "0020_clean_scheduled_jobs"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            forward_change,
+            revert_change,
+        ),
+    ]

{ipfabric_netbox-4.3.2b1 → ipfabric_netbox-4.3.2b3}/ipfabric_netbox/models.py

@@ -2,6 +2,7 @@ import ast
 import functools
 import json
 import logging
+import re
 import traceback
 from copy import deepcopy
 from uuid import uuid4
@@ -12,10 +13,8 @@ from core.exceptions import SyncError
 from core.models import Job
 from core.models import ObjectType
 from core.signals import pre_sync
-from dcim.models import Device
+from dcim.models import MACAddress
 from dcim.models import Site
-from dcim.models import VirtualChassis
-from dcim.signals import assign_virtualchassis_master
 from django.apps import apps
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
@@ -50,7 +49,7 @@ from .choices import IPFabricSourceTypeChoices
 from .choices import IPFabricSyncStatusChoices
 from .choices import IPFabricTransformMapSourceModelChoices
 from .choices import required_transform_map_contenttypes
-from .signals import clear_other_primary_ip
+from .signals import assign_primary_mac_address
 from .utilities.ipfutils import IPFabric
 from .utilities.ipfutils import IPFabricSyncRunner
 from .utilities.ipfutils import render_jinja2
@@ -229,6 +228,22 @@ class IPFabricTransformMap(NetBoxModel):
                         rel_dict_coalesce[field.target_field] = related_object
         return rel_dict, rel_dict_coalesce
 
+    def strip_source_data(self, source_data: dict) -> dict:
+        """Strip data according to Transform Map mappings but without rendering templates."""
+        keys = set()
+        for field in self.field_maps.all():
+            keys.add(field.source_field)
+            if field.template:
+                keys.update(
+                    re.findall(r"object\.([a-zA-Z_0-9]+)(?=.*)", field.template)
+                )
+        for field in self.relationship_maps.all():
+            if field.template:
+                keys.update(
+                    re.findall(r"object\.([a-zA-Z_0-9]+)(?=.*)", field.template)
+                )
+        return {k: source_data[k] for k in keys}
+
     def get_context(self, source_data):
         new_data = deepcopy(source_data)
         relationship, coalesce_relationship = self.build_relationships(
@@ -241,7 +256,8 @@ class IPFabricTransformMap(NetBoxModel):
         context = self.render(new_data)
         return context
 
-    def update_or_create_instance(self, context, tags=[], connection_name=None):
+    def update_or_create_instance(self, context, tags=None, connection_name=None):
+        tags = tags or []
         target_class = self.target_model.model_class()
         queryset = target_class.objects.using(connection_name)
 
@@ -272,8 +288,11 @@ class IPFabricTransformMap(NetBoxModel):
                     if field.endswith("__isnull"):
                         context[field[:-8]] = None
                         del context[field]
-                instance = queryset.create(**context, **defaults)
+                # Using queryset.create() creates the object even when it fails on clean()
+                # To to work around it, we do it in two steps to avoid saving it to DB before clean()
+                instance = queryset.model(**context, **defaults)
                 instance.full_clean()
+                instance.save(using=connection_name)
 
             apply_tags(instance, tags, connection_name)
 
@@ -867,9 +886,17 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
             try:
                 active_branch.set(branch)
                 try:
-                    runner.collect_and_sync(
-                        ingestion=IPFabricIngestion.objects.get(pk=ingestion.pk)
-                    )
+                    try:
+                        signals.post_save.connect(
+                            assign_primary_mac_address, sender=MACAddress
+                        )
+                        runner.collect_and_sync(
+                            ingestion=IPFabricIngestion.objects.get(pk=ingestion.pk)
+                        )
+                    finally:
+                        signals.post_save.disconnect(
+                            assign_primary_mac_address, sender=MACAddress
+                        )
                 finally:
                     active_branch.set(None)
             finally:
@@ -1002,15 +1029,7 @@ class IPFabricIngestion(JobsMixin, models.Model):
         # Begin Sync
         logger.debug(f"Merging {self.name}")
         try:
-            signals.pre_save.connect(clear_other_primary_ip, sender=Device)
-            signals.post_save.disconnect(
-                assign_virtualchassis_master, sender=VirtualChassis
-            )
             self.branch.merge(user=self.sync.user)
-            signals.post_save.connect(
-                assign_virtualchassis_master, sender=VirtualChassis
-            )
-            signals.pre_save.disconnect(clear_other_primary_ip, sender=Device)
             ipfabricsync.status = IPFabricSyncStatusChoices.COMPLETED
         except Exception as e:
             ipfabricsync.status = IPFabricSyncStatusChoices.FAILED

ipfabric_netbox-4.3.2b3/ipfabric_netbox/signals.py

@@ -0,0 +1,41 @@
+import logging
+
+from dcim.models import Interface
+from dcim.models import MACAddress
+from netbox_branching.contextvars import active_branch
+
+logger = logging.getLogger("ipfabric_netbox.utilities.ipf_utils")
+
+
+def assign_primary_mac_address(instance: MACAddress, **kwargs) -> None:
+    try:
+        if instance.assigned_object and instance.assigned_object.primary_mac_address:
+            # The Interface already has primary MAC, nothing to do
+            return
+    except Interface.DoesNotExist:
+        # The Interface is not created yet, cannot be assigned
+        return
+
+    connection_name = None
+    if branch := active_branch.get():
+        connection_name = branch.connection_name
+    instance.assigned_object.snapshot()
+    instance.assigned_object.primary_mac_address = instance
+    instance.assigned_object.save(using=connection_name)
+
+
+def remove_group_from_syncs(instance, **kwargs):
+    """
+    When an IPFabricTransformMapGroup is deleted, remove its ID from any IPFabricSync.parameters['groups'] list.
+    """
+    from ipfabric_netbox.models import IPFabricSync
+
+    group_id = instance.pk
+    for sync in IPFabricSync.objects.all():
+        params = sync.parameters or {}
+        groups = params.get("groups", [])
+        if group_id not in groups:
+            continue
+        params["groups"] = [gid for gid in groups if gid != group_id]
+        sync.parameters = params
+        sync.save()

{ipfabric_netbox-4.3.2b1 → ipfabric_netbox-4.3.2b3}/ipfabric_netbox/templates/ipfabric_netbox/inc/diff.html

@@ -9,7 +9,7 @@
     <div class="row mb-3">
         <div class="col-6">
             <div class="card">
-                <h5 class="card-header">%{ trans "Pre-Change Data" %}</h5>
+                <h5 class="card-header">{% trans "Pre-Change Data" %}</h5>
                 <div class="card-body">
                     {% if prechange_data %}
                     <pre class="change-data">{% for k, v in prechange_data.items %}{% spaceless %}
@@ -17,14 +17,14 @@
                     {% endspaceless %}{% endfor %}
                 </pre>
                     {% else %}
-                    <span class="text-muted">%{ trans "None" %}</span>
+                    <span class="text-muted">{% trans "None" %}</span>
                     {% endif %}
                 </div>
             </div>
         </div>
         <div class="col-6">
             <div class="card">
-                <h5 class="card-header">%{ trans "Post-Change Data" %}</h5>
+                <h5 class="card-header">{% trans "Post-Change Data" %}</h5>
                 <div class="card-body">
                     {% if postchange_data %}
                     <pre class="change-data">{% for k, v in postchange_data.items %}{% spaceless %}
@@ -38,7 +38,7 @@
         <div class="col">
             <div class="card">
                 <div class="card-header">
-                    <h5 class="d-inline">%{ trans "Difference" %}</h5>
+                    <h5 class="d-inline">{% trans "Difference" %}</h5>
                     <!-- <div class="btn-group btn-group-sm float-end noprint">
                             <a {% if prev_change %}href="{% url 'extras:objectchange' pk=prev_change.pk %}"{% else %}disabled{% endif %} class="btn btn-outline-secondary">
                                 <i class="mdi mdi-chevron-left" aria-hidden="true"></i> Previous
@@ -52,11 +52,11 @@
                     {% if diff_added == diff_removed %}
                     <span class="text-muted" style="margin-left: 10px;">
                         {% if object.action == 'create' %}
-                        %{ trans "Object Created" %}
+                        {% trans "Object Created" %}
                         {% elif object.action == 'delete' %}
-                        %{ trans "Object Deleted" %}
+                        {% trans "Object Deleted" %}
                         {% else %}
-                        %{ trans "No Changes" %}
+                        {% trans "No Changes" %}
                         {% endif %}
                     </span>
                     {% else %}

{ipfabric_netbox-4.3.2b1 → ipfabric_netbox-4.3.2b3}/ipfabric_netbox/tests/test_models.py

@@ -174,7 +174,10 @@ class IPFabricTransformMapModelTestCase(TestCase):
         }
 
         self.site = runner.sync_item(
-            item=site_data, app_label="dcim", model="site", cf=sync.update_custom_fields
+            record=runner.create_new_data_record(
+                app="dcim", model="site", data=site_data
+            ),
+            cf=sync.update_custom_fields,
         )
 
         device_data = {
@@ -210,37 +213,38 @@ class IPFabricTransformMapModelTestCase(TestCase):
             "uptime": 7254180,
             "vendor": "cisco",
             "version": "15.2(4)M1",
+            "virtual_chassis": None,
             "slug": None,
         }
 
         self.mf_obj = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="manufacturer",
+            record=runner.create_new_data_record(
+                app="dcim", model="manufacturer", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.dt_obj = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="devicetype",
+            record=runner.create_new_data_record(
+                app="dcim", model="devicetype", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.platform = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="platform",
+            record=runner.create_new_data_record(
+                app="dcim", model="platform", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.role = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="devicerole",
+            record=runner.create_new_data_record(
+                app="dcim", model="devicerole", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.device_object = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="device",
+            record=runner.create_new_data_record(
+                app="dcim", model="device", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
 
@@ -401,6 +405,7 @@ class IPFabricTransformMapModelTestCase(TestCase):
             "uptime": 7254180,
             "vendor": "cisco",
             "version": "15.2(4)M1",
+            "virtual_chassis": None,
             "slug": None,
         }
 
@@ -415,9 +420,9 @@ class IPFabricTransformMapModelTestCase(TestCase):
         transform_field.template = "{{ object.hostname }} - test"
         transform_field.save()
         device_object = runner.sync_item(
-            item=device_data,
-            app_label="dcim",
-            model="device",
+            record=runner.create_new_data_record(
+                app="dcim", model="device", data=device_data
+            ),
             cf=sync.update_custom_fields,
         )
         self.assertEqual(device_object.name, "L21PE152 - test")