ipfabric_netbox 4.1.0b4__tar.gz → 4.1.0b6__tar.gz

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ipfabric_netbox
-Version: 4.1.0b4
+Version: 4.1.0b6
 Summary: NetBox plugin to sync IP Fabric data into NetBox
 License: MIT
 Keywords: netbox,ipfabric,plugin,sync

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/__init__.py

@@ -6,7 +6,7 @@ class NetboxIPFabricConfig(PluginConfig):
     name = "ipfabric_netbox"
     verbose_name = "NetBox IP Fabric SoT Plugin"
     description = "Sync IP Fabric into NetBox"
-    version = "4.1.0b4"
+    version = "4.1.0b6"
     base_url = "ipfabric"
     min_version = "4.2.4"
 

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/api/nested_serializers.py

@@ -10,7 +10,6 @@ from ipfabric_netbox.models import IPFabricSource
 from ipfabric_netbox.models import IPFabricSync
 from ipfabric_netbox.models import IPFabricTransformMap
 
-# from .serializers import IPFabricSyncSerializer
 
 __all__ = (
     "NestedIPFabricSourceSerializer",

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/api/serializers.py

@@ -7,8 +7,10 @@ from rest_framework import serializers
 
 from .nested_serializers import NestedIPFabricSnapshotSerializer
 from .nested_serializers import NestedIPFabricSourceSerializer
+from .nested_serializers import NestedIPFabricSyncSerializer
 from .nested_serializers import NestedIPFabricTransformMapSerializer
 from ipfabric_netbox.models import IPFabricIngestion
+from ipfabric_netbox.models import IPFabricIngestionIssue
 from ipfabric_netbox.models import IPFabricRelationshipField
 from ipfabric_netbox.models import IPFabricSnapshot
 from ipfabric_netbox.models import IPFabricSource
@@ -23,7 +25,9 @@ __all__ = (
     "IPFabricRelationshipFieldSerializer",
     "IPFabricTransformFieldSerializer",
     "IPFabricTransformMapSerializer",
+    "IPFabricTransformMapGroupSerializer",
     "IPFabricIngestionSerializer",
+    "IPFabricIngestionIssueSerializer",
     "IPFabricSourceSerializer",
 )
 
@@ -133,7 +137,7 @@ class IPFabricTransformFieldSerializer(NetBoxModelSerializer):
 
 class IPFabricIngestionSerializer(NetBoxModelSerializer):
     branch = BranchSerializer(read_only=True)
-    sync = IPFabricSyncSerializer(read_only=True)
+    sync = NestedIPFabricSyncSerializer(read_only=True)
 
     class Meta:
         model = IPFabricIngestion
@@ -145,6 +149,24 @@ class IPFabricIngestionSerializer(NetBoxModelSerializer):
         ]
 
 
+class IPFabricIngestionIssueSerializer(NetBoxModelSerializer):
+    ingestion = IPFabricIngestionSerializer(read_only=True)
+
+    class Meta:
+        model = IPFabricIngestionIssue
+        fields = [
+            "id",
+            "ingestion",
+            "timestamp",
+            "model",
+            "message",
+            "raw_data",
+            "coalesce_fields",
+            "defaults",
+            "exception",
+        ]
+
+
 class IPFabricSourceSerializer(NetBoxModelSerializer):
     status = ChoiceField(choices=DataSourceStatusChoices)
     url = serializers.URLField()

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/api/urls.py

@@ -1,6 +1,7 @@
 # api/urls.py
 from netbox.api.routers import NetBoxRouter
 
+from ipfabric_netbox.api.views import IPFabricIngestionIssueViewSet
 from ipfabric_netbox.api.views import IPFabricIngestionViewSet
 from ipfabric_netbox.api.views import IPFabricRelationshipFieldiewSet
 from ipfabric_netbox.api.views import IPFabricSnapshotViewSet
@@ -18,6 +19,7 @@ router.register("transform-map-group", IPFabricTransformMapGroupViewSet)
 router.register("transform-map", IPFabricTransformMapViewSet)
 router.register("sync", IPFabricSyncViewSet)
 router.register("ingestion", IPFabricIngestionViewSet)
+router.register("ingestion-issues", IPFabricIngestionIssueViewSet)
 router.register("transform-field", IPFabricTransformFieldiewSet)
 router.register("relationship-field", IPFabricRelationshipFieldiewSet)
 urlpatterns = router.urls

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/api/views.py

@@ -5,6 +5,7 @@ from rest_framework.decorators import action
 from rest_framework.response import Response
 from utilities.query import count_related
 
+from .serializers import IPFabricIngestionIssueSerializer
 from .serializers import IPFabricIngestionSerializer
 from .serializers import IPFabricRelationshipFieldSerializer
 from .serializers import IPFabricSnapshotSerializer
@@ -18,6 +19,7 @@ from ipfabric_netbox.filtersets import IPFabricSourceFilterSet
 from ipfabric_netbox.filtersets import IPFabricTransformFieldFilterSet
 from ipfabric_netbox.models import IPFabricData
 from ipfabric_netbox.models import IPFabricIngestion
+from ipfabric_netbox.models import IPFabricIngestionIssue
 from ipfabric_netbox.models import IPFabricRelationshipField
 from ipfabric_netbox.models import IPFabricSnapshot
 from ipfabric_netbox.models import IPFabricSource
@@ -59,6 +61,11 @@ class IPFabricIngestionViewSet(NetBoxReadOnlyModelViewSet):
     serializer_class = IPFabricIngestionSerializer
 
 
+class IPFabricIngestionIssueViewSet(NetBoxReadOnlyModelViewSet):
+    queryset = IPFabricIngestionIssue.objects.all()
+    serializer_class = IPFabricIngestionIssueSerializer
+
+
 class IPFabricSnapshotViewSet(NetBoxModelViewSet):
     queryset = IPFabricSnapshot.objects.all()
     serializer_class = IPFabricSnapshotSerializer

ipfabric_netbox-4.1.0b6/ipfabric_netbox/exceptions.py

@@ -0,0 +1,81 @@
+from typing import TYPE_CHECKING
+
+from core.exceptions import SyncError
+
+if TYPE_CHECKING:
+    from .models import IPFabricIngestionIssue
+    from .models import IPFabricIngestion
+
+
+class IngestionIssue(Exception):
+    """
+    This exception is used to indicate an issue during the ingestion process.
+    """
+
+    # Store created issue object ID if it exists for this exception
+    issue_id = None
+    model: str = ""
+    defaults: dict[str, str] = {}
+    coalesce_fields: dict[str, str] = {}
+
+    def __init__(self, model: str, data: dict, context: dict = None, issue_id=None):
+        super().__init__()
+        self.model = model
+        self.data = data
+        context = context or {}
+        self.defaults = context.pop("defaults", {})
+        self.coalesce_fields = context
+        self.issue_id = issue_id
+
+
+class SearchError(IngestionIssue, LookupError):
+    def __init__(self, message: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class SyncDataError(IngestionIssue, SyncError):
+    def __str__(self):
+        return f"Sync failed for {self.model}: coalesce_fields={self.coalesce_fields} defaults={self.defaults}."
+
+
+class IPAddressDuplicateError(IngestionIssue, SyncError):
+    def __str__(self):
+        return f"IP address {self.data.get('address')} already exists in {self.model} with coalesce_fields={self.coalesce_fields}."
+
+
+def create_or_get_sync_issue(
+    exception: Exception,
+    ingestion: "IPFabricIngestion",
+    message: str = None,
+    model: str = None,
+    context: dict = None,
+    data: dict = None,
+) -> (bool, "IPFabricIngestionIssue"):
+    """
+    Helper function to handle sync errors and create IPFabricIngestionIssue if needed.
+    """
+    context = context or {}
+
+    # TODO: This is to prevent circular import issues, clean it up later.
+    from .models import IPFabricIngestionIssue
+
+    if not hasattr(exception, "issue_id") or not exception.issue_id:
+        issue = IPFabricIngestionIssue.objects.create(
+            ingestion=ingestion,
+            exception=exception.__class__.__name__,
+            message=message or getattr(exception, "message", str(exception)),
+            model=model,
+            coalesce_fields={k: v for k, v in context.items() if k not in ["defaults"]},
+            defaults=context.get("defaults", dict()),
+            raw_data=data or dict(),
+        )
+        if hasattr(exception, "issue_id"):
+            exception.issue_id = issue.id
+        return True, issue
+    else:
+        issue = IPFabricIngestionIssue.objects.get(id=exception.issue_id)
+        return False, issue

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/filtersets.py

@@ -10,6 +10,7 @@ from netbox_branching.models import ChangeDiff
 
 from .models import IPFabricData
 from .models import IPFabricIngestion
+from .models import IPFabricIngestionIssue
 from .models import IPFabricSnapshot
 from .models import IPFabricSource
 from .models import IPFabricSync
@@ -37,6 +38,35 @@ class IPFabricIngestionChangeFilterSet(BaseFilterSet):
         )
 
 
+class IPFabricIngestionIssueFilterSet(BaseFilterSet):
+    q = django_filters.CharFilter(method="search")
+
+    class Meta:
+        model = IPFabricIngestionIssue
+        fields = [
+            "model",
+            "timestamp",
+            "raw_data",
+            "coalesce_fields",
+            "defaults",
+            "exception",
+            "message",
+        ]
+
+    def search(self, queryset, name, value):
+        if not value.strip():
+            return queryset
+        return queryset.filter(
+            Q(model__icontains=value)
+            | Q(timestamp__icontains=value)
+            | Q(raw_data__icontains=value)
+            | Q(coalesce_fields__icontains=value)
+            | Q(defaults__icontains=value)
+            | Q(exception__icontains=value)
+            | Q(message__icontains=value)
+        )
+
+
 class IPFabricDataFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(method="search")
 

ipfabric_netbox-4.1.0b6/ipfabric_netbox/migrations/0015_ipfabricingestionissue.py

@@ -0,0 +1,48 @@
+# Generated by Django 5.2 on 2025-07-11 19:39
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "ipfabric_netbox",
+            "0014_ipfabrictransformmapgroup_ipfabrictransformmap_group",
+        ),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="IPFabricIngestionIssue",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False
+                    ),
+                ),
+                ("timestamp", models.DateTimeField(default=django.utils.timezone.now)),
+                ("model", models.CharField(blank=True, max_length=100, null=True)),
+                ("message", models.TextField()),
+                ("raw_data", models.TextField(blank=True, default="")),
+                ("coalesce_fields", models.TextField(blank=True, default="")),
+                ("defaults", models.TextField(blank=True, default="")),
+                ("exception", models.TextField()),
+                (
+                    "ingestion",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="issues",
+                        to="ipfabric_netbox.ipfabricingestion",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "IP Fabric Ingestion Issue",
+                "verbose_name_plural": "IP Fabric Ingestion Issues",
+                "ordering": ["timestamp"],
+            },
+        ),
+    ]

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/models.py

@@ -236,6 +236,8 @@ class IPFabricTransformMap(NetBoxModel):
         target_class = self.target_model.model_class()
         queryset = target_class.objects.using(connection_name)
 
+        # Don't change context since it's used in case of exception for IPFabricIngestionIssue
+        context = deepcopy(context)
         defaults = context.pop("defaults", {})
 
         with transaction.atomic(using=connection_name):
@@ -967,6 +969,29 @@ class IPFabricIngestion(JobsMixin, models.Model):
         )
 
 
+class IPFabricIngestionIssue(models.Model):
+    objects = RestrictedQuerySet.as_manager()
+
+    ingestion = models.ForeignKey(
+        to="IPFabricIngestion", on_delete=models.CASCADE, related_name="issues"
+    )
+    timestamp = models.DateTimeField(default=timezone.now)
+    model = models.CharField(max_length=100, blank=True, null=True)
+    message = models.TextField()
+    raw_data = models.TextField(blank=True, default="")
+    coalesce_fields = models.TextField(blank=True, default="")
+    defaults = models.TextField(blank=True, default="")
+    exception = models.TextField()
+
+    class Meta:
+        ordering = ["timestamp"]
+        verbose_name = "IP Fabric Ingestion Issue"
+        verbose_name_plural = "IP Fabric Ingestion Issues"
+
+    def __str__(self):
+        return f"[{self.timestamp}] {self.message}"
+
+
 class IPFabricData(models.Model):
     snapshot_data = models.ForeignKey(
         to=IPFabricSnapshot,

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/tables.py

@@ -8,6 +8,7 @@ from netbox_branching.models import ChangeDiff
 
 from .models import IPFabricData
 from .models import IPFabricIngestion
+from .models import IPFabricIngestionIssue
 from .models import IPFabricRelationshipField
 from .models import IPFabricSnapshot
 from .models import IPFabricSource
@@ -199,6 +200,28 @@ class IPFabricIngestionChangesTable(NetBoxTable):
         default_columns = ("object", "action", "object_type", "actions")
 
 
+class IPFabricIngestionIssuesTable(NetBoxTable):
+    id = tables.Column(verbose_name=_("ID"))
+    exception = tables.Column(verbose_name="Exception Type")
+    message = tables.Column(verbose_name="Error Message")
+    actions = None
+
+    class Meta(NetBoxTable.Meta):
+        model = IPFabricIngestionIssue
+        fields = (
+            "model",
+            "timestamp",
+            "raw_data",
+            "coalesce_fields",
+            "defaults",
+            "exception",
+            "message",
+        )
+        default_columns = ("model", "exception", "message")
+        empty_text = _("No Ingestion Issues found")
+        order_by = "id"
+
+
 class DeviceIPFTable(tables.Table):
     hostname = Column()
 

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/templates/ipfabric_netbox/ipfabricingestion.html

@@ -36,6 +36,11 @@
 </div>
 {% endblock control-buttons %}
 
+{% block tabs %}
+  <div id="object_tabs" hx-swap-oob="true">
+    {% include 'ipfabric_netbox/partials/object_tabs.html' with object=object %}
+  </div>
+{% endblock tabs %}
 
 {% block content %}
 <div class="row mb-3">
@@ -70,7 +75,6 @@
             <th scope="row">Ingestion Status</th>
               <td><div id="ingestion_status" hx-swap-oob="true">{% include 'ipfabric_netbox/partials/ingestion_status.html' with object=object %}</div></td>
               <!-- <td >{% badge object.sync.get_status_display bg_color=object.sync.get_status_color %}</td> -->
-            </div>
           </tr>
           <tr>
             <th scope="row">Snapshot</th>
@@ -88,59 +92,26 @@
     {% plugin_left_page object %}
   </div>
   <div class="col col-md-6">
-    <div class="card">
-      <h5 class="card-header">Statistics</h5>
-      <div class="card-body">
-        <table class="table table-hover attr-table">
-          <tr>
-            <th scope="row">Created</th>
-            <td>
-              {% if object.num_created %}
-              {{ object.num_created }}
-              {% else %}
-              {{ ''|placeholder }}
-              {% endif %}
-            </td>
-          </tr>
-          <tr>
-            <th scope="row">Updated</th>
-            <td>
-              {% if object.num_updated %}
-              {{ object.num_updated }}
-              {% else %}
-              {{ ''|placeholder }}
-              {% endif %}
-            </td>
-          </tr>
-          <tr>
-            <th scope="row">Deleted</th>
-            <td>
-              {% if object.num_deleted %}
-              {{ object.num_deleted }}
-              {% else %}
-              {{ ''|placeholder }}
-              {% endif %}
-            </td>
-          </tr>
-        </table>
-      </div>
+    <div id="ingestion_statistics" hx-swap-oob="true">
+      {% include 'ipfabric_netbox/partials/ingestion_statistics.html' with object=object %}
     </div>
-    <div id="ingestion_progress">
+    <div id="ingestion_progress" hx-swap-oob="true">
       {% include 'ipfabric_netbox/partials/ingestion_progress.html' %}
     </div>
     <!-- {% include 'inc/panels/related_objects.html' %} -->
     {% include 'inc/panels/custom_fields.html' %}
     {% plugin_right_page object %}
   </div>
-</div>
-<div class="row mb-3">
-  <div class="col col-md-12" {% if not object.job.completed %} hx-get="{% url 'plugins:ipfabric_netbox:ipfabricingestion_logs' pk=object.pk %}?type=info"
-    hx-trigger="every 5s" hx-target="#ingestion_logs" hx-select="#ingestion_logs" hx-select-oob="#ingestion_status:innerHTML"
-    hx-swap="innerHTML" {% endif %}>
-    <div id="ingestion_logs">{% include 'ipfabric_netbox/partials/job_logs.html' with job=object.job %}</div>
+  <div class="row mb-3">
+    <div class="col col-md-12" {% if not object.job.completed %} hx-get="{% url 'plugins:ipfabric_netbox:ipfabricingestion_logs' pk=object.pk %}?type=info"
+      hx-trigger="every 5s" hx-target="#ingestion_logs" hx-select="#ingestion_logs" hx-select-oob="#ingestion_status:innerHTML,#ingestion_progress:innerHTML,#ingestion_statistics:innerHTML,#object_tabs:innerHTML"
+      hx-swap="innerHTML" {% endif %}>
+      <div id="ingestion_logs">{% include 'ipfabric_netbox/partials/job_logs.html' with job=object.job %}</div>
+    </div>
   </div>
+</div>
 {% endblock content %}
 
-  {% block modals %}
-  {% include 'inc/htmx_modal.html' with size='lg' %}
-  {% endblock modals %}
+{% block modals %}
+{% include 'inc/htmx_modal.html' with size='lg' %}
+{% endblock modals %}

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/templates/ipfabric_netbox/partials/ingestion_all.html

@@ -1,4 +1,9 @@
-
+<div id="object_tabs" hx-swap-oob="true">
+  {% include 'ipfabric_netbox/partials/object_tabs.html' with object=object %}
+</div>
+<div id="ingestion_statistics" hx-swap-oob="true">
+  {% include 'ipfabric_netbox/partials/ingestion_statistics.html' with object=object %}
+</div>
 <div id="ingestion_status" hx-swap-oob="true">
   {% include 'ipfabric_netbox/partials/ingestion_status.html' with object=object %}
 </div>

ipfabric_netbox-4.1.0b6/ipfabric_netbox/templates/ipfabric_netbox/partials/ingestion_statistics.html

@@ -0,0 +1,37 @@
+<div class="card">
+  <h5 class="card-header">Statistics</h5>
+  <div class="card-body">
+    <table class="table table-hover attr-table">
+      <tr>
+        <th scope="row">Created</th>
+        <td>
+          {% if object.num_created %}
+          {{ object.num_created }}
+          {% else %}
+          {{ ''|placeholder }}
+          {% endif %}
+        </td>
+      </tr>
+      <tr>
+        <th scope="row">Updated</th>
+        <td>
+          {% if object.num_updated %}
+          {{ object.num_updated }}
+          {% else %}
+          {{ ''|placeholder }}
+          {% endif %}
+        </td>
+      </tr>
+      <tr>
+        <th scope="row">Deleted</th>
+        <td>
+          {% if object.num_deleted %}
+          {{ object.num_deleted }}
+          {% else %}
+          {{ ''|placeholder }}
+          {% endif %}
+        </td>
+      </tr>
+    </table>
+  </div>
+</div>

ipfabric_netbox-4.1.0b6/ipfabric_netbox/templates/ipfabric_netbox/partials/ingestion_status.html

@@ -0,0 +1 @@
+{% badge object.sync.get_status_display bg_color=object.sync.get_status_color %} {% if object.errors and object.errors.count %}(with {{ object.errors.count }} issues){% endif %}

ipfabric_netbox-4.1.0b6/ipfabric_netbox/templates/ipfabric_netbox/partials/object_tabs.html

@@ -0,0 +1,12 @@
+{% load helpers %}
+{% load tabs %}
+
+<ul class="nav nav-tabs" role="presentation">
+  {# Primary tab #}
+  <li class="nav-item">
+    <a class="nav-link{% if not tab %} active{% endif %}" href="{{ object.get_absolute_url }}">{{ object|meta:"verbose_name"|bettertitle }}</a>
+  </li>
+
+  {# Include tabs for registered model views #}
+  {% model_view_tabs object %}
+</ul>

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/utilities/ipfutils.py

@@ -22,19 +22,14 @@ from netbox.config import get_config
 from netutils.utils import jinja2_convenience_function
 
 from ..choices import IPFabricSourceTypeChoices
+from ..exceptions import create_or_get_sync_issue
+from ..exceptions import IPAddressDuplicateError
 from ..exceptions import SearchError
 from ..exceptions import SyncDataError
 from .nbutils import device_serial_max_length
 from .nbutils import order_devices
 from .nbutils import order_members
 
-FAILED_CREATE_INSTANCE_TEMPLATE = (
-    "Failed to create instance of '{model}': <br/>"
-    "message: '{err}'<br/>"
-    "raw data: '{data}'<br/>"
-    "context: '{context}'<br/>"
-)
-
 if TYPE_CHECKING:
     from ..models import IPFabricIngestion
     from ipam.models import IPAddress
@@ -163,6 +158,9 @@ class IPFabricSyncRunner(object):
             except Exception as err:
                 # Log the error to logger outside of job - console/file
                 logger.error(err, exc_info=True)
+                if hasattr(err, "issue_id") and err.issue_id:
+                    # The error is already logged to user, no need to log it again
+                    return None
                 # Logging section for logs inside job - facing user
                 self = args[0]
                 if isinstance(err, SearchError):
@@ -176,6 +174,11 @@ class IPFabricSyncRunner(object):
                             f"Syncing `{err.model}` is disabled in settings, but hit above error trying to find the correct item. Please check your transform maps and/or existing data.",
                             obj=self.sync,
                         )
+                if isinstance(err, IPAddressDuplicateError):
+                    self.logger.log_warning(
+                        f"IP Address `{err.data.get('address')}` already exists in `{err.model}` with coalesce fields: `{err.coalesce_fields}`. Please check your transform maps and/or existing data.",
+                        obj=self.sync,
+                    )
                 else:
                     self.logger.log_failure(
                         f"Syncing failed with: `{err}`. See above error for more details.",
@@ -207,13 +210,21 @@ class IPFabricSyncRunner(object):
         except Exception as err:
             message = f"Error getting context for `{model}`."
             if isinstance(err, ObjectDoesNotExist):
-                message += " Could not find related object using on template in transform maps."
+                message += (
+                    " Could not find related object using template in transform maps."
+                )
             elif isinstance(err, MultipleObjectsReturned):
                 message += " Multiple objects returned using on template in transform maps, the template is not strict enough."
-            self.logger.log_failure(
-                f"<b>{message}</b><br/>data: `{data}`<br/>error: `{err}`", obj=self.sync
+            _, issue = create_or_get_sync_issue(
+                exception=err,
+                ingestion=self.ingestion,
+                message=message,
+                model=model,
+                data=data,
             )
-            raise SyncError(err) from err
+            raise SearchError(
+                message=message, data=data, model=model, issue_id=issue.id
+            ) from err
 
         queryset = transform_map.target_model.model_class().objects
 
@@ -232,25 +243,50 @@ class IPFabricSyncRunner(object):
                 context.pop("defaults", None)
                 object = queryset.using(connection_name).get(**context)
         except queryset.model.DoesNotExist as err:
-            self.logger.log_failure(
-                f"<b>`{model}` with these keys not found: `{context}`.</b><br/>Original data: `{data}`.",
-                obj=self.sync,
+            message = f"Instance of `{model}` not found."
+            _, issue = create_or_get_sync_issue(
+                exception=err,
+                ingestion=self.ingestion,
+                message=message,
+                model=model,
+                context=context,
+                data=data,
             )
-            raise SearchError(model=model, context=context, data=data) from err
+            raise SearchError(
+                message=message,
+                model=model,
+                context=context,
+                data=data,
+                issue_id=issue.id,
+            ) from err
         except queryset.model.MultipleObjectsReturned as err:
-            self.logger.log_failure(
-                f"<b>Multiple `{model}` with these keys found: `{context}`.</b><br/>Original data: `{data}`.",
-                obj=self.sync,
+            message = f"Multiple instances of `{model}` found."
+            _, issue = create_or_get_sync_issue(
+                exception=err,
+                ingestion=self.ingestion,
+                message=message,
+                model=model,
+                context=context,
+                data=data,
             )
-            raise SearchError(model=model, context=context, data=data) from err
+            raise SearchError(
+                message=message,
+                model=model,
+                context=context,
+                data=data,
+                issue_id=issue.id,
+            ) from err
         except Exception as err:
-            self.logger.log_failure(
-                FAILED_CREATE_INSTANCE_TEMPLATE.format(
-                    model=model, err=err, data=data, context=context
-                ),
-                obj=self.sync,
+            _, issue = create_or_get_sync_issue(
+                exception=err,
+                ingestion=self.ingestion,
+                model=model,
+                context=context,
+                data=data,
             )
-            raise SyncDataError(model=model, context=context, data=data) from err
+            raise SyncDataError(
+                model=model, context=context, data=data, issue_id=issue.id
+            ) from err
 
         return object
 
@@ -440,7 +476,6 @@ class IPFabricSyncRunner(object):
             f"{len(data.get('ipaddress', []))} management IP's collected",
             obj=self.sync.snapshot_data.source,
         )
-
         self.logger.log_info("Ordering devices", obj=self.sync)
 
         members = order_members(data.get("virtualchassis", []))
@@ -631,7 +666,7 @@ class IPFabricSyncRunner(object):
                 "dcim",
                 "virtualchassis",
                 virtual_chassis,
-                False,
+                stats=False,
                 sync=self.settings.get("virtualchassis"),
             )
 
@@ -640,6 +675,7 @@ class IPFabricSyncRunner(object):
                 "dcim",
                 "device",
                 device,
+                stats=False,
                 sync=self.settings.get("device")
                 or self.settings.get("interface")
                 or self.settings.get("ipaddress")
@@ -666,7 +702,7 @@ class IPFabricSyncRunner(object):
                     "dcim",
                     "virtualchassis",
                     virtual_chassis,
-                    False,
+                    stats=False,
                     sync=self.settings.get("virtualchassis"),
                 )
 

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/ipfabric_netbox/views.py

@@ -34,6 +34,7 @@ from utilities.views import ViewTab
 from .filtersets import IPFabricDataFilterSet
 from .filtersets import IPFabricIngestionChangeFilterSet
 from .filtersets import IPFabricIngestionFilterSet
+from .filtersets import IPFabricIngestionIssueFilterSet
 from .filtersets import IPFabricSnapshotFilterSet
 from .filtersets import IPFabricSourceFilterSet
 from .filtersets import IPFabricTransformMapFilterSet
@@ -52,6 +53,7 @@ from .forms import IPFabricTransformMapForm
 from .forms import IPFabricTransformMapGroupForm
 from .models import IPFabricData
 from .models import IPFabricIngestion
+from .models import IPFabricIngestionIssue
 from .models import IPFabricRelationshipField
 from .models import IPFabricSnapshot
 from .models import IPFabricSource
@@ -62,6 +64,7 @@ from .models import IPFabricTransformMapGroup
 from .tables import DeviceIPFTable
 from .tables import IPFabricDataTable
 from .tables import IPFabricIngestionChangesTable
+from .tables import IPFabricIngestionIssuesTable
 from .tables import IPFabricIngestionTable
 from .tables import IPFabricRelationshipFieldTable
 from .tables import IPFabricSnapshotTable
@@ -657,36 +660,8 @@ class IPFabricIngestionListView(generic.ObjectListView):
     table = IPFabricIngestionTable
 
 
-@register_model_view(
-    IPFabricIngestion,
-    name="logs",
-    path="logs",
-)
-class IPFabricIngestionLogView(LoginRequiredMixin, View):
-    template_name = "ipfabric_netbox/partials/ingestion_all.html"
-
-    def get(self, request, **kwargs):
-        ingestion_id = kwargs.get("pk")
-        if request.htmx:
-            ingestion = IPFabricIngestion.objects.get(pk=ingestion_id)
-            data = ingestion.get_statistics()
-            data["object"] = ingestion
-            data["job"] = ingestion.jobs.first()
-            response = render(
-                request,
-                self.template_name,
-                data,
-            )
-            if ingestion.job.completed:
-                response["HX-Refresh"] = "true"
-                return response
-            else:
-                return response
-
-
-@register_model_view(IPFabricIngestion)
-class IPFabricIngestionView(generic.ObjectView):
-    queryset = IPFabricIngestion.objects.annotate(
+def annotate_statistics(queryset):
+    return queryset.annotate(
         num_created=models.Count(
             "branch__changediff",
             filter=models.Q(
@@ -713,6 +688,39 @@ class IPFabricIngestionView(generic.ObjectView):
         staged_changes=models.Count(models.F("branch__changediff")),
     )
 
+
+@register_model_view(
+    IPFabricIngestion,
+    name="logs",
+    path="logs",
+)
+class IPFabricIngestionLogView(LoginRequiredMixin, View):
+    template_name = "ipfabric_netbox/partials/ingestion_all.html"
+
+    def get(self, request, **kwargs):
+        ingestion_id = kwargs.get("pk")
+        if request.htmx:
+            ingestion = annotate_statistics(IPFabricIngestion.objects).get(
+                pk=ingestion_id
+            )
+            data = ingestion.get_statistics()
+            data["object"] = ingestion
+            data["job"] = ingestion.jobs.first()
+            response = render(
+                request,
+                self.template_name,
+                data,
+            )
+            if ingestion.job.completed:
+                response["HX-Refresh"] = "true"
+            return response
+        return render(request, self.template_name)
+
+
+@register_model_view(IPFabricIngestion)
+class IPFabricIngestionView(generic.ObjectView):
+    queryset = annotate_statistics(IPFabricIngestion.objects)
+
     def get_extra_context(self, request, instance):
         data = instance.get_statistics()
         return data
@@ -844,6 +852,23 @@ class IPFabricIngestionChangesView(generic.ObjectChildrenView):
         return self.child_model.objects.filter(branch=parent.branch)
 
 
+@register_model_view(IPFabricIngestion, "ingestion_issues")
+class IPFabricIngestionIssuesView(generic.ObjectChildrenView):
+    queryset = IPFabricIngestion.objects.all()
+    child_model = IPFabricIngestionIssue
+    table = IPFabricIngestionIssuesTable
+    template_name = "generic/object_children.html"
+    filterset = IPFabricIngestionIssueFilterSet
+    tab = ViewTab(
+        label="Ingestion Issues",
+        badge=lambda obj: IPFabricIngestionIssue.objects.filter(ingestion=obj).count(),
+        permission="ipfabric_netbox.view_ipfabricingestionissue",
+    )
+
+    def get_children(self, request, parent):
+        return IPFabricIngestionIssue.objects.filter(ingestion=parent)
+
+
 @register_model_view(IPFabricIngestion, "delete")
 class IPFabricIngestionDeleteView(generic.ObjectDeleteView):
     queryset = IPFabricIngestion.objects.all()

{ipfabric_netbox-4.1.0b4 → ipfabric_netbox-4.1.0b6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ipfabric_netbox"
-version = "4.1.0b4"
+version = "4.1.0b6"
 description = "NetBox plugin to sync IP Fabric data into NetBox"
 authors = ["Solution Architecture <solution.architecture@ipfabric.io>"]
 license = "MIT"

ipfabric_netbox-4.1.0b4/ipfabric_netbox/exceptions.py

@@ -1,24 +0,0 @@
-from core.exceptions import SyncError
-
-
-class ErrorMixin(Exception):
-    model: str = ""
-    defaults: dict[str, str] = {}
-    coalesce_fields: dict[str, str] = {}
-
-    def __init__(self, model: str, context: dict, data: dict = None):
-        super().__init__()
-        self.model = model
-        self.data = data or {}
-        self.defaults = context.pop("defaults", {})
-        self.coalesce_fields = context
-
-
-class SearchError(ErrorMixin, LookupError):
-    def __str__(self):
-        return f"{self.model} with these keys not found: {self.coalesce_fields}."
-
-
-class SyncDataError(ErrorMixin, SyncError):
-    def __str__(self):
-        return f"Sync failed for {self.model}: coalesce_fields={self.coalesce_fields} defaults={self.defaults}."

ipfabric_netbox-4.1.0b4/ipfabric_netbox/templates/ipfabric_netbox/partials/ingestion_status.html

@@ -1 +0,0 @@
-{% badge object.sync.get_status_display bg_color=object.sync.get_status_color %}