ipfabric_netbox 4.1.0b3__tar.gz → 4.1.0b5__tar.gz

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ipfabric_netbox
-Version: 4.1.0b3
+Version: 4.1.0b5
 Summary: NetBox plugin to sync IP Fabric data into NetBox
 License: MIT
 Keywords: netbox,ipfabric,plugin,sync

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/__init__.py

@@ -6,7 +6,7 @@ class NetboxIPFabricConfig(PluginConfig):
     name = "ipfabric_netbox"
     verbose_name = "NetBox IP Fabric SoT Plugin"
     description = "Sync IP Fabric into NetBox"
-    version = "4.1.0b3"
+    version = "4.1.0b5"
     base_url = "ipfabric"
     min_version = "4.2.4"
 

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/api/nested_serializers.py

@@ -10,7 +10,6 @@ from ipfabric_netbox.models import IPFabricSource
 from ipfabric_netbox.models import IPFabricSync
 from ipfabric_netbox.models import IPFabricTransformMap
 
-# from .serializers import IPFabricSyncSerializer
 
 __all__ = (
     "NestedIPFabricSourceSerializer",

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/api/serializers.py

@@ -7,8 +7,10 @@ from rest_framework import serializers
 
 from .nested_serializers import NestedIPFabricSnapshotSerializer
 from .nested_serializers import NestedIPFabricSourceSerializer
+from .nested_serializers import NestedIPFabricSyncSerializer
 from .nested_serializers import NestedIPFabricTransformMapSerializer
 from ipfabric_netbox.models import IPFabricIngestion
+from ipfabric_netbox.models import IPFabricIngestionIssue
 from ipfabric_netbox.models import IPFabricRelationshipField
 from ipfabric_netbox.models import IPFabricSnapshot
 from ipfabric_netbox.models import IPFabricSource
@@ -23,7 +25,9 @@ __all__ = (
     "IPFabricRelationshipFieldSerializer",
     "IPFabricTransformFieldSerializer",
     "IPFabricTransformMapSerializer",
+    "IPFabricTransformMapGroupSerializer",
     "IPFabricIngestionSerializer",
+    "IPFabricIngestionIssueSerializer",
     "IPFabricSourceSerializer",
 )
 
@@ -133,7 +137,7 @@ class IPFabricTransformFieldSerializer(NetBoxModelSerializer):
 
 class IPFabricIngestionSerializer(NetBoxModelSerializer):
     branch = BranchSerializer(read_only=True)
-    sync = IPFabricSyncSerializer(read_only=True)
+    sync = NestedIPFabricSyncSerializer(read_only=True)
 
     class Meta:
         model = IPFabricIngestion
@@ -145,6 +149,24 @@ class IPFabricIngestionSerializer(NetBoxModelSerializer):
         ]
 
 
+class IPFabricIngestionIssueSerializer(NetBoxModelSerializer):
+    ingestion = IPFabricIngestionSerializer(read_only=True)
+
+    class Meta:
+        model = IPFabricIngestionIssue
+        fields = [
+            "id",
+            "ingestion",
+            "timestamp",
+            "model",
+            "message",
+            "raw_data",
+            "coalesce_fields",
+            "defaults",
+            "exception",
+        ]
+
+
 class IPFabricSourceSerializer(NetBoxModelSerializer):
     status = ChoiceField(choices=DataSourceStatusChoices)
     url = serializers.URLField()

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/api/urls.py

@@ -1,6 +1,7 @@
 # api/urls.py
 from netbox.api.routers import NetBoxRouter
 
+from ipfabric_netbox.api.views import IPFabricIngestionIssueViewSet
 from ipfabric_netbox.api.views import IPFabricIngestionViewSet
 from ipfabric_netbox.api.views import IPFabricRelationshipFieldiewSet
 from ipfabric_netbox.api.views import IPFabricSnapshotViewSet
@@ -18,6 +19,7 @@ router.register("transform-map-group", IPFabricTransformMapGroupViewSet)
 router.register("transform-map", IPFabricTransformMapViewSet)
 router.register("sync", IPFabricSyncViewSet)
 router.register("ingestion", IPFabricIngestionViewSet)
+router.register("ingestion-issues", IPFabricIngestionIssueViewSet)
 router.register("transform-field", IPFabricTransformFieldiewSet)
 router.register("relationship-field", IPFabricRelationshipFieldiewSet)
 urlpatterns = router.urls

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/api/views.py

@@ -5,6 +5,7 @@ from rest_framework.decorators import action
 from rest_framework.response import Response
 from utilities.query import count_related
 
+from .serializers import IPFabricIngestionIssueSerializer
 from .serializers import IPFabricIngestionSerializer
 from .serializers import IPFabricRelationshipFieldSerializer
 from .serializers import IPFabricSnapshotSerializer
@@ -18,6 +19,7 @@ from ipfabric_netbox.filtersets import IPFabricSourceFilterSet
 from ipfabric_netbox.filtersets import IPFabricTransformFieldFilterSet
 from ipfabric_netbox.models import IPFabricData
 from ipfabric_netbox.models import IPFabricIngestion
+from ipfabric_netbox.models import IPFabricIngestionIssue
 from ipfabric_netbox.models import IPFabricRelationshipField
 from ipfabric_netbox.models import IPFabricSnapshot
 from ipfabric_netbox.models import IPFabricSource
@@ -59,6 +61,11 @@ class IPFabricIngestionViewSet(NetBoxReadOnlyModelViewSet):
     serializer_class = IPFabricIngestionSerializer
 
 
+class IPFabricIngestionIssueViewSet(NetBoxReadOnlyModelViewSet):
+    queryset = IPFabricIngestionIssue.objects.all()
+    serializer_class = IPFabricIngestionIssueSerializer
+
+
 class IPFabricSnapshotViewSet(NetBoxModelViewSet):
     queryset = IPFabricSnapshot.objects.all()
     serializer_class = IPFabricSnapshotSerializer

ipfabric_netbox-4.1.0b5/ipfabric_netbox/exceptions.py

@@ -0,0 +1,81 @@
+from typing import TYPE_CHECKING
+
+from core.exceptions import SyncError
+
+if TYPE_CHECKING:
+    from .models import IPFabricIngestionIssue
+    from .models import IPFabricIngestion
+
+
+class IngestionIssue(Exception):
+    """
+    This exception is used to indicate an issue during the ingestion process.
+    """
+
+    # Store created issue object ID if it exists for this exception
+    issue_id = None
+    model: str = ""
+    defaults: dict[str, str] = {}
+    coalesce_fields: dict[str, str] = {}
+
+    def __init__(self, model: str, data: dict, context: dict = None, issue_id=None):
+        super().__init__()
+        self.model = model
+        self.data = data
+        context = context or {}
+        self.defaults = context.pop("defaults", {})
+        self.coalesce_fields = context
+        self.issue_id = issue_id
+
+
+class SearchError(IngestionIssue, LookupError):
+    def __init__(self, message: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class SyncDataError(IngestionIssue, SyncError):
+    def __str__(self):
+        return f"Sync failed for {self.model}: coalesce_fields={self.coalesce_fields} defaults={self.defaults}."
+
+
+class IPAddressDuplicateError(IngestionIssue, SyncError):
+    def __str__(self):
+        return f"IP address {self.data.get('address')} already exists in {self.model} with coalesce_fields={self.coalesce_fields}."
+
+
+def create_or_get_sync_issue(
+    exception: Exception,
+    ingestion: "IPFabricIngestion",
+    message: str = None,
+    model: str = None,
+    context: dict = None,
+    data: dict = None,
+) -> (bool, "IPFabricIngestionIssue"):
+    """
+    Helper function to handle sync errors and create IPFabricIngestionIssue if needed.
+    """
+    context = context or {}
+
+    # TODO: This is to prevent circular import issues, clean it up later.
+    from .models import IPFabricIngestionIssue
+
+    if not hasattr(exception, "issue_id") or not exception.issue_id:
+        issue = IPFabricIngestionIssue.objects.create(
+            ingestion=ingestion,
+            exception=exception.__class__.__name__,
+            message=message or getattr(exception, "message", str(exception)),
+            model=model,
+            coalesce_fields={k: v for k, v in context.items() if k not in ["defaults"]},
+            defaults=context.get("defaults", dict()),
+            raw_data=data or dict(),
+        )
+        if hasattr(exception, "issue_id"):
+            exception.issue_id = issue.id
+        return True, issue
+    else:
+        issue = IPFabricIngestionIssue.objects.get(id=exception.issue_id)
+        return False, issue

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/filtersets.py

@@ -10,6 +10,7 @@ from netbox_branching.models import ChangeDiff
 
 from .models import IPFabricData
 from .models import IPFabricIngestion
+from .models import IPFabricIngestionIssue
 from .models import IPFabricSnapshot
 from .models import IPFabricSource
 from .models import IPFabricSync
@@ -37,6 +38,35 @@ class IPFabricIngestionChangeFilterSet(BaseFilterSet):
         )
 
 
+class IPFabricIngestionIssueFilterSet(BaseFilterSet):
+    q = django_filters.CharFilter(method="search")
+
+    class Meta:
+        model = IPFabricIngestionIssue
+        fields = [
+            "model",
+            "timestamp",
+            "raw_data",
+            "coalesce_fields",
+            "defaults",
+            "exception",
+            "message",
+        ]
+
+    def search(self, queryset, name, value):
+        if not value.strip():
+            return queryset
+        return queryset.filter(
+            Q(model__icontains=value)
+            | Q(timestamp__icontains=value)
+            | Q(raw_data__icontains=value)
+            | Q(coalesce_fields__icontains=value)
+            | Q(defaults__icontains=value)
+            | Q(exception__icontains=value)
+            | Q(message__icontains=value)
+        )
+
+
 class IPFabricDataFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(method="search")
 

ipfabric_netbox-4.1.0b5/ipfabric_netbox/migrations/0015_ipfabricingestionissue.py

@@ -0,0 +1,48 @@
+# Generated by Django 5.2 on 2025-07-11 19:39
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "ipfabric_netbox",
+            "0014_ipfabrictransformmapgroup_ipfabrictransformmap_group",
+        ),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="IPFabricIngestionIssue",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False
+                    ),
+                ),
+                ("timestamp", models.DateTimeField(default=django.utils.timezone.now)),
+                ("model", models.CharField(blank=True, max_length=100, null=True)),
+                ("message", models.TextField()),
+                ("raw_data", models.TextField(blank=True, default="")),
+                ("coalesce_fields", models.TextField(blank=True, default="")),
+                ("defaults", models.TextField(blank=True, default="")),
+                ("exception", models.TextField()),
+                (
+                    "ingestion",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="issues",
+                        to="ipfabric_netbox.ipfabricingestion",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "IP Fabric Ingestion Issue",
+                "verbose_name_plural": "IP Fabric Ingestion Issues",
+                "ordering": ["timestamp"],
+            },
+        ),
+    ]

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/models.py

@@ -236,6 +236,8 @@ class IPFabricTransformMap(NetBoxModel):
         target_class = self.target_model.model_class()
         queryset = target_class.objects.using(connection_name)
 
+        # Don't change context since it's used in case of exception for IPFabricIngestionIssue
+        context = deepcopy(context)
         defaults = context.pop("defaults", {})
 
         with transaction.atomic(using=connection_name):
@@ -967,6 +969,29 @@ class IPFabricIngestion(JobsMixin, models.Model):
         )
 
 
+class IPFabricIngestionIssue(models.Model):
+    objects = RestrictedQuerySet.as_manager()
+
+    ingestion = models.ForeignKey(
+        to="IPFabricIngestion", on_delete=models.CASCADE, related_name="issues"
+    )
+    timestamp = models.DateTimeField(default=timezone.now)
+    model = models.CharField(max_length=100, blank=True, null=True)
+    message = models.TextField()
+    raw_data = models.TextField(blank=True, default="")
+    coalesce_fields = models.TextField(blank=True, default="")
+    defaults = models.TextField(blank=True, default="")
+    exception = models.TextField()
+
+    class Meta:
+        ordering = ["timestamp"]
+        verbose_name = "IP Fabric Ingestion Issue"
+        verbose_name_plural = "IP Fabric Ingestion Issues"
+
+    def __str__(self):
+        return f"[{self.timestamp}] {self.message}"
+
+
 class IPFabricData(models.Model):
     snapshot_data = models.ForeignKey(
         to=IPFabricSnapshot,

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/tables.py

@@ -8,6 +8,7 @@ from netbox_branching.models import ChangeDiff
 
 from .models import IPFabricData
 from .models import IPFabricIngestion
+from .models import IPFabricIngestionIssue
 from .models import IPFabricRelationshipField
 from .models import IPFabricSnapshot
 from .models import IPFabricSource
@@ -199,6 +200,28 @@ class IPFabricIngestionChangesTable(NetBoxTable):
         default_columns = ("object", "action", "object_type", "actions")
 
 
+class IPFabricIngestionIssuesTable(NetBoxTable):
+    id = tables.Column(verbose_name=_("ID"))
+    exception = tables.Column(verbose_name="Exception Type")
+    message = tables.Column(verbose_name="Error Message")
+    actions = None
+
+    class Meta(NetBoxTable.Meta):
+        model = IPFabricIngestionIssue
+        fields = (
+            "model",
+            "timestamp",
+            "raw_data",
+            "coalesce_fields",
+            "defaults",
+            "exception",
+            "message",
+        )
+        default_columns = ("model", "exception", "message")
+        empty_text = _("No Ingestion Issues found")
+        order_by = "id"
+
+
 class DeviceIPFTable(tables.Table):
     hostname = Column()
 

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/templates/ipfabric_netbox/ipfabricingestion.html

@@ -36,6 +36,11 @@
 </div>
 {% endblock control-buttons %}
 
+{% block tabs %}
+  <div id="object_tabs" hx-swap-oob="true">
+    {% include 'ipfabric_netbox/partials/object_tabs.html' with object=object %}
+  </div>
+{% endblock tabs %}
 
 {% block content %}
 <div class="row mb-3">
@@ -70,7 +75,6 @@
             <th scope="row">Ingestion Status</th>
               <td><div id="ingestion_status" hx-swap-oob="true">{% include 'ipfabric_netbox/partials/ingestion_status.html' with object=object %}</div></td>
               <!-- <td >{% badge object.sync.get_status_display bg_color=object.sync.get_status_color %}</td> -->
-            </div>
           </tr>
           <tr>
             <th scope="row">Snapshot</th>
@@ -88,59 +92,26 @@
     {% plugin_left_page object %}
   </div>
   <div class="col col-md-6">
-    <div class="card">
-      <h5 class="card-header">Statistics</h5>
-      <div class="card-body">
-        <table class="table table-hover attr-table">
-          <tr>
-            <th scope="row">Created</th>
-            <td>
-              {% if object.num_created %}
-              {{ object.num_created }}
-              {% else %}
-              {{ ''|placeholder }}
-              {% endif %}
-            </td>
-          </tr>
-          <tr>
-            <th scope="row">Updated</th>
-            <td>
-              {% if object.num_updated %}
-              {{ object.num_updated }}
-              {% else %}
-              {{ ''|placeholder }}
-              {% endif %}
-            </td>
-          </tr>
-          <tr>
-            <th scope="row">Deleted</th>
-            <td>
-              {% if object.num_deleted %}
-              {{ object.num_deleted }}
-              {% else %}
-              {{ ''|placeholder }}
-              {% endif %}
-            </td>
-          </tr>
-        </table>
-      </div>
+    <div id="ingestion_statistics" hx-swap-oob="true">
+      {% include 'ipfabric_netbox/partials/ingestion_statistics.html' with object=object %}
     </div>
-    <div id="ingestion_progress">
+    <div id="ingestion_progress" hx-swap-oob="true">
       {% include 'ipfabric_netbox/partials/ingestion_progress.html' %}
     </div>
     <!-- {% include 'inc/panels/related_objects.html' %} -->
     {% include 'inc/panels/custom_fields.html' %}
     {% plugin_right_page object %}
   </div>
-</div>
-<div class="row mb-3">
-  <div class="col col-md-12" {% if not object.job.completed %} hx-get="{% url 'plugins:ipfabric_netbox:ipfabricingestion_logs' pk=object.pk %}?type=info"
-    hx-trigger="every 5s" hx-target="#ingestion_logs" hx-select="#ingestion_logs" hx-select-oob="#ingestion_status:innerHTML"
-    hx-swap="innerHTML" {% endif %}>
-    <div id="ingestion_logs">{% include 'ipfabric_netbox/partials/job_logs.html' with job=object.job %}</div>
+  <div class="row mb-3">
+    <div class="col col-md-12" {% if not object.job.completed %} hx-get="{% url 'plugins:ipfabric_netbox:ipfabricingestion_logs' pk=object.pk %}?type=info"
+      hx-trigger="every 5s" hx-target="#ingestion_logs" hx-select="#ingestion_logs" hx-select-oob="#ingestion_status:innerHTML,#ingestion_progress:innerHTML,#ingestion_statistics:innerHTML,#object_tabs:innerHTML"
+      hx-swap="innerHTML" {% endif %}>
+      <div id="ingestion_logs">{% include 'ipfabric_netbox/partials/job_logs.html' with job=object.job %}</div>
+    </div>
   </div>
+</div>
 {% endblock content %}
 
-  {% block modals %}
-  {% include 'inc/htmx_modal.html' with size='lg' %}
-  {% endblock modals %}
+{% block modals %}
+{% include 'inc/htmx_modal.html' with size='lg' %}
+{% endblock modals %}

{ipfabric_netbox-4.1.0b3 → ipfabric_netbox-4.1.0b5}/ipfabric_netbox/templates/ipfabric_netbox/partials/ingestion_all.html

@@ -1,4 +1,9 @@
-
+<div id="object_tabs" hx-swap-oob="true">
+  {% include 'ipfabric_netbox/partials/object_tabs.html' with object=object %}
+</div>
+<div id="ingestion_statistics" hx-swap-oob="true">
+  {% include 'ipfabric_netbox/partials/ingestion_statistics.html' with object=object %}
+</div>
 <div id="ingestion_status" hx-swap-oob="true">
   {% include 'ipfabric_netbox/partials/ingestion_status.html' with object=object %}
 </div>

ipfabric_netbox-4.1.0b5/ipfabric_netbox/templates/ipfabric_netbox/partials/ingestion_statistics.html

@@ -0,0 +1,37 @@
+<div class="card">
+  <h5 class="card-header">Statistics</h5>
+  <div class="card-body">
+    <table class="table table-hover attr-table">
+      <tr>
+        <th scope="row">Created</th>
+        <td>
+          {% if object.num_created %}
+          {{ object.num_created }}
+          {% else %}
+          {{ ''|placeholder }}
+          {% endif %}
+        </td>
+      </tr>
+      <tr>
+        <th scope="row">Updated</th>
+        <td>
+          {% if object.num_updated %}
+          {{ object.num_updated }}
+          {% else %}
+          {{ ''|placeholder }}
+          {% endif %}
+        </td>
+      </tr>
+      <tr>
+        <th scope="row">Deleted</th>
+        <td>
+          {% if object.num_deleted %}
+          {{ object.num_deleted }}
+          {% else %}
+          {{ ''|placeholder }}
+          {% endif %}
+        </td>
+      </tr>
+    </table>
+  </div>
+</div>

ipfabric_netbox-4.1.0b5/ipfabric_netbox/templates/ipfabric_netbox/partials/ingestion_status.html

@@ -0,0 +1 @@
+{% badge object.sync.get_status_display bg_color=object.sync.get_status_color %} {% if object.errors and object.errors.count %}(with {{ object.errors.count }} issues){% endif %}

ipfabric_netbox-4.1.0b5/ipfabric_netbox/templates/ipfabric_netbox/partials/object_tabs.html

@@ -0,0 +1,12 @@
+{% load helpers %}
+{% load tabs %}
+
+<ul class="nav nav-tabs" role="presentation">
+  {# Primary tab #}
+  <li class="nav-item">
+    <a class="nav-link{% if not tab %} active{% endif %}" href="{{ object.get_absolute_url }}">{{ object|meta:"verbose_name"|bettertitle }}</a>
+  </li>
+
+  {# Include tabs for registered model views #}
+  {% model_view_tabs object %}
+</ul>