ipfabric_netbox 3.2.4__py3-none-any.whl → 3.2.4b2__py3-none-any.whl

ipfabric_netbox/tables.py

@@ -1,12 +1,13 @@
 import django_tables2 as tables
+from django.utils.html import format_html
 from django.utils.translation import gettext_lazy as _
 from django_tables2 import Column
-from extras.models import StagedChange
 from netbox.tables import columns
 from netbox.tables import NetBoxTable
+from netbox_branching.models import ChangeDiff
 
-from .models import IPFabricBranch
 from .models import IPFabricData
+from .models import IPFabricIngestion
 from .models import IPFabricRelationshipField
 from .models import IPFabricSnapshot
 from .models import IPFabricSource
@@ -17,7 +18,7 @@ from .models import IPFabricTransformMap
 
 DIFF_BUTTON = """
     <a href="#"
-          hx-get="{% url 'plugins:ipfabric_netbox:ipfabricbranch_change_diff' pk=record.branch.pk change_pk=record.pk %}"
+          hx-get="{% url 'plugins:ipfabric_netbox:ipfabricingestion_change_diff' pk=record.branch.pk change_pk=record.pk %}"
           hx-target="#htmx-modal-content"
           data-bs-toggle="modal"
           data-bs-target="#htmx-modal"
@@ -64,23 +65,21 @@ class IPFabricTransformMapTable(NetBoxTable):
 
     class Meta(NetBoxTable.Meta):
         model = IPFabricTransformMap
-        fields = ("name", "source_model", "target_model", "status")
-        default_columns = ("name", "source_model", "target_model", "status")
+        fields = ("name", "source_model", "target_model")
+        default_columns = ("name", "source_model", "target_model")
 
 
-class BranchTable(NetBoxTable):
+class IPFabricIngestionTable(NetBoxTable):
     name = tables.Column(linkify=True)
     sync = tables.Column(verbose_name="IP Fabric Sync", linkify=True)
+    branch = tables.Column(linkify=True)
     changes = tables.Column(accessor="staged_changes", verbose_name="Number of Changes")
     actions = columns.ActionsColumn(actions=("delete",))
 
-    def render_changes(self, value):
-        return value.count()
-
     class Meta(NetBoxTable.Meta):
-        model = IPFabricBranch
-        fields = ("name", "sync", "description", "user", "changes")
-        default_columns = ("name", "sync", "description", "user", "changes")
+        model = IPFabricIngestion
+        fields = ("name", "sync", "branch", "description", "user", "changes")
+        default_columns = ("name", "sync", "branch", "description", "user", "changes")
 
 
 class IPFabricSnapshotTable(NetBoxTable):
@@ -142,51 +141,46 @@ class SyncTable(NetBoxTable):
         default_columns = ("id", "status", "snapshot_name")
 
 
-class StagedChangesTable(NetBoxTable):
-    # There is no view for single StagedChange, remove the link in ID
+class IPFabricIngestionChangesTable(NetBoxTable):
+    # There is no view for single change, remove the link in ID
     id = tables.Column(verbose_name=_("ID"))
     pk = None
     object_type = tables.Column(
         accessor="object_type.model", verbose_name="Object Type"
     )
-    name = tables.Column(accessor="object_type.name", verbose_name="Name")
+    object = tables.Column(verbose_name="Object")
     actions = columns.TemplateColumn(template_code=DIFF_BUTTON)
 
-    def render_name(self, value, record):
-        models_with_names = [
-            "site",
-            "manufacturer",
-            "device role",
-            "device",
-            "interface",
-            "platform",
-            "inventory item",
-            "VLAN",
-        ]
-        if record.data:
-            if value in models_with_names:
-                name = record.data["name"]
-            elif value == "device type":
-                name = record.data["model"]
-            elif value == "IP address":
-                name = record.data["address"]
-            elif value == "tagged item":
-                name = f"Tagging object ({record.data['object_id']})"
-            elif value == "prefix":
-                name = f"{record.data['prefix']} ({record.data['vrf']})"
-            elif value == "MAC address":
-                name = record.data["mac_address"]
-            else:
-                name = record.data
+    def render_object(self, value, record):
+        model_templates = {
+            "Device": lambda v: v.name,
+            "DeviceRole": lambda v: v.name,
+            "DeviceType": lambda v: v.model,
+            "IPAddress": lambda v: v.address,
+            "Interface": lambda v: f"{v.name} (Device {v.device.name})",
+            "InventoryItem": lambda v: f"{v.name} (Device {v.device.name})",
+            "MACAddress": lambda v: v.mac_address,
+            "Manufacturer": lambda v: v.name,
+            "Platform": lambda v: v.name,
+            "Prefix": lambda v: f"{v.prefix} (VRF {v.vrf})",
+            "Site": lambda v: v.name,
+            "VirtualChassis": lambda v: v.name,
+            "VLAN": lambda v: f"{v.name} (VID {v.vid})",
+            "VRF": lambda v: v.name,
+        }
+        if value and (class_name := value.__class__.__name__) in model_templates:
+            field_value = model_templates[class_name](value)
+            if url := value.get_absolute_url():
+                return format_html("<a href={}>{}</a>", url, field_value)
         else:
-            name = record.object.__str__()
-        return name
+            field_value = record.object_repr
+        return field_value
 
     class Meta(NetBoxTable.Meta):
-        model = StagedChange
+        model = ChangeDiff
         name = "staged_changes"
-        fields = ("name", "action", "object_type", "actions")
-        default_columns = ("name", "action", "object_type", "actions")
+        fields = ("object", "action", "object_type", "actions")
+        default_columns = ("object", "action", "object_type", "actions")
 
 
 class DeviceIPFTable(tables.Table):

ipfabric_netbox/templates/ipfabric_netbox/{ipfabricbranch.html → ipfabricingestion.html}

@@ -17,7 +17,7 @@
 {% if perms.core.sync_datasource %}
 {% if object.sync.ready_for_sync %}
   <a href="#"
-  hx-get="{% url 'plugins:ipfabric_netbox:ipfabricbranch_merge' pk=object.pk %}"
+  hx-get="{% url 'plugins:ipfabric_netbox:ipfabricingestion_merge' pk=object.pk %}"
   hx-target="#htmx-modal-content"
   class="btn btn-success"
   data-bs-toggle="modal"
@@ -41,7 +41,7 @@
 <div class="row mb-3">
   <div class="col col-md-6">
     <div class="card">
-      <h5 class="card-header">Branch Information</h5>
+      <h5 class="card-header">Ingestion Information</h5>
       <div class="card-body">
         <table class="table table-hover attr-table">
           <tr>
@@ -57,13 +57,18 @@
             <td><a href="{% url 'plugins:ipfabric_netbox:ipfabricsync' pk=object.sync.pk %}">{{ object.sync.name}}</a>
             </td>
           </tr>
+          <tr>
+            <th scope="row">Branch Object</th>
+            <td>{% if object.branch %}<a href="{% url 'plugins:netbox_branching:branch' pk=object.branch.pk %}">{{ object.branch.name}}</a>{% else %}{{ ""|placeholder }}{% endif %}
+            </td>
+          </tr>
           <tr>
             <th scope="row">Job Object</th>
             <td><a href="{% url 'core:job' pk=object.job.pk %}">{{ object.job }}</a></td>
           </tr>
           <tr>
-            <th scope="row">Sync Status</th>
-              <td><div id="sync_status" hx-swap-oob="true">{% include 'ipfabric_netbox/partials/branch_status.html' with object=object %}</div></td>
+            <th scope="row">Ingestion Status</th>
+              <td><div id="ingestion_status" hx-swap-oob="true">{% include 'ipfabric_netbox/partials/ingestion_status.html' with object=object %}</div></td>
               <!-- <td >{% badge object.sync.get_status_display bg_color=object.sync.get_status_color %}</td> -->
             </div>
           </tr>
@@ -120,8 +125,8 @@
         </table>
       </div>
     </div>
-    <div id="sync_progress">
-      {% include 'ipfabric_netbox/partials/branch_progress.html' %}
+    <div id="ingestion_progress">
+      {% include 'ipfabric_netbox/partials/ingestion_progress.html' %}
     </div>
     <!-- {% include 'inc/panels/related_objects.html' %} -->
     {% include 'inc/panels/custom_fields.html' %}
@@ -129,10 +134,10 @@
   </div>
 </div>
 <div class="row mb-3">
-  <div class="col col-md-12" {% if not object.job.completed %} hx-get="{% url 'plugins:ipfabric_netbox:ipfabricbranch_logs' pk=object.pk %}?type=info"
-    hx-trigger="every 5s" hx-target="#sync_logs" hx-select="#sync_logs" hx-select-oob="#sync_status:innerHTML"
+  <div class="col col-md-12" {% if not object.job.completed %} hx-get="{% url 'plugins:ipfabric_netbox:ipfabricingestion_logs' pk=object.pk %}?type=info"
+    hx-trigger="every 5s" hx-target="#ingestion_logs" hx-select="#ingestion_logs" hx-select-oob="#ingestion_status:innerHTML"
     hx-swap="innerHTML" {% endif %}>
-    <div id="sync_logs">{% include 'ipfabric_netbox/partials/job_logs.html' with job=object.job %}</div>
+    <div id="ingestion_logs">{% include 'ipfabric_netbox/partials/job_logs.html' with job=object.job %}</div>
   </div>
 {% endblock content %}
 

ipfabric_netbox/templates/ipfabric_netbox/ipfabricsource.html

@@ -102,9 +102,9 @@
     {% plugin_right_page object %}
   </div>
   {% if job %}
-  <div class="col col-md-12" {% if not object.job.completed %} hx-trigger="every 5s" hx-target="#sync_logs"
-    hx-select="#sync_logs" hx-select-oob="#sync_status:innerHTML" hx-swap="innerHTML" {% endif %}>
-    <div id="sync_logs">{% include 'ipfabric_netbox/partials/job_logs.html' with job=job %}</div>
+  <div class="col col-md-12" {% if not object.job.completed %} hx-trigger="every 5s" hx-target="#ingestion_logs"
+    hx-select="#ingestion_logs" hx-select-oob="#ingestion_status:innerHTML" hx-swap="innerHTML" {% endif %}>
+    <div id="ingestionlogs">{% include 'ipfabric_netbox/partials/job_logs.html' with job=job %}</div>
   </div>
   {% endif %}
 </div>

ipfabric_netbox/templates/ipfabric_netbox/ipfabricsync.html

@@ -5,7 +5,7 @@
 {% load render_table from django_tables2 %}
 
 {% block extra_controls %}
-  {% if perms.ipfabric_netbox.sync_ingest %}
+  {% if perms.ipfabric_netbox.start_sync %}
     {% if object.ready_for_sync %}
       <form action="{% url 'plugins:ipfabric_netbox:ipfabricsync_sync' pk=object.pk %}" method="post">
         {% csrf_token %}
@@ -51,8 +51,8 @@
             <td>{{ object.last_synced|placeholder }}</td>
           </tr>
           <tr>
-            <th scope="row">Latest branch</th>
-            <td><div {% if object.status == 'queued' or object.status == 'syncing'  %} hx-get="{% url 'plugins:ipfabric_netbox:ipfabricsync' pk=object.pk %}" hx-trigger="every 1s"{% endif %}>{% include 'ipfabric_netbox/partials/sync_last_branch.html' %}</div></td>
+            <th scope="row">Latest ingestion</th>
+            <td><div {% if object.status == 'queued' or object.status == 'syncing'  %} hx-get="{% url 'plugins:ipfabric_netbox:ipfabricsync' pk=object.pk %}" hx-trigger="every 1s"{% endif %}>{% include 'ipfabric_netbox/partials/sync_last_ingestion.html' %}</div></td>
           </tr>
           <tr>
             <th scope="row">Schedule</th>

ipfabric_netbox/templates/ipfabric_netbox/ipfabricsync_list.html

@@ -0,0 +1,71 @@
+{% extends 'base/layout.html' %}
+{% load buttons %}
+{% load helpers %}
+{% load perms %}
+
+{% block title %}Ingestion{% endblock %}
+
+{% block tabs %}
+  <ul class="nav nav-tabs px-3">
+    <li class="nav-item" role="presentation">
+      <a class="nav-link active" role="tab">Ingestion</a>
+    </li>
+  </ul>
+{% endblock tabs %}
+
+{% block controls %}
+  <div class="controls">
+    <div class="control-group">
+      {% block extra_controls %}{% endblock %}
+      {% add_button model %}
+    </div>
+  </div>
+{% endblock controls %}
+
+{% block content %}
+  <div class="tab-content">
+    {% for sync in syncs %}
+      <div class="card">
+        <h5 class="card-header d-flex justify-content-between" id="module{{ module.pk }}">
+          <div>
+          <i class="mdi mdi-cloud-sync"></i><i class="mdi mdi-sync"></i> <a href="{% url 'plugins:ipfabric_netbox:ipfabricsync' pk=sync.pk %}">{{ sync.name}}</a>
+          </div>
+          {% if perms.ipfabric_netbox.delete_ipfabricsync %}
+              <a href="{% url 'plugins:ipfabric_netbox:ipfabricsync_delete' pk=sync.pk %}" class="btn btn-danger btn-sm">
+                <i class="mdi mdi-trash-can-outline" aria-hidden="true"></i> Delete
+              </a>
+          {% endif %}
+
+        </h5>
+        <div class="card-body">
+          {% include 'inc/sync_warning.html' with object=module %}
+          <table class="table table-hover table-headings reports">
+            <thead>
+              <tr>
+                <th width="200">Source</th>
+                <th width="400">Snapshot</th>
+                <th>Status</th>
+                <th>Last Run</th>
+              </tr>
+            </thead>
+            <tbody>
+              <tr>
+                <td><a href="{% url 'plugins:ipfabric_netbox:ipfabricsource' pk=sync.snapshot_data.source.pk %}">{{ sync.snapshot_data.source.name }}</a></td>
+                <td><a href="{% url 'plugins:ipfabric_netbox:ipfabricsnapshot' pk=sync.snapshot_data.pk %}">{{ sync.snapshot_data.name}}</a></td>
+                <td>{% badge sync.get_status_display last_job.get_status_color %}</td>
+                <td>{{sync.last_synced}}</td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+      </div>
+    {% empty %}
+      <div class="alert alert-info" role="alert">
+        <h4 class="alert-heading">Sync Jobs Settings Found</h4>
+        {% if perms.extras.add_reportmodule %}
+          Get started by <a href="{% url 'plugins:ipfabric_netbox:ipfabricsync_add' %}">creating a sync</a> from an IP Fabric source.
+        {% endif %}
+      </div>
+    {% endfor %}
+  </div>
+{% endblock content %}

ipfabric_netbox/templates/ipfabric_netbox/ipfabrictransformmap.html

@@ -23,10 +23,6 @@
             <th scope="row">Target Model</th>
             <td>{{ object.target_model }}</td>
           </tr>
-          <tr>
-            <th scope="row">Status</th>
-            <td>{{ object.status }}</td>
-          </tr>
           </tr>
         </table>
       </div>

ipfabric_netbox/templates/ipfabric_netbox/partials/ingestion_all.html

@@ -0,0 +1,10 @@
+
+<div id="ingestion_status" hx-swap-oob="true">
+  {% include 'ipfabric_netbox/partials/ingestion_status.html' with object=object %}
+</div>
+<div id="ingestion_progress" hx-swap-oob="true">
+  {% include 'ipfabric_netbox/partials/ingestion_progress.html' %}
+</div>
+<div id="ingestion_logs">
+  {% include 'ipfabric_netbox/partials/job_logs.html' with job_results=job_results job=job %}
+</div>

ipfabric_netbox/templates/ipfabric_netbox/partials/{branch_progress.html → ingestion_progress.html}

@@ -12,7 +12,7 @@
             {% endfor %}
         </table>
         {% else %}
-        Sync progress pending...<br>
+        Ingestion progress pending...<br>
         <small class="text-muted">Last updated {% now "H:i:s" %}</small>
         {% endif %}
     </div>

ipfabric_netbox/templates/ipfabric_netbox/partials/sync_last_ingestion.html

@@ -0,0 +1 @@
+{% if last_ingestion %}<a href="{% url 'plugins:ipfabric_netbox:ipfabricingestion' pk=last_ingestion.pk %}">{{ last_ingestion.name }}{% else %}{{ ""|placeholder }}{% endif %}</a>

ipfabric_netbox/urls.py

@@ -63,10 +63,20 @@ urlpatterns = (
     path("sync/", views.IPFabricSyncListView.as_view(), name="ipfabricsync_list"),
     path("sync/add/", views.IPFabricSyncEditView.as_view(), name="ipfabricsync_add"),
     path("sync/<int:pk>/", include(get_model_urls("ipfabric_netbox", "ipfabricsync"))),
-    # Branch
-    path("branch/", views.IPFabricBranchListView.as_view(), name="ipfabricbranch_list"),
     path(
-        "branch/<int:pk>/", include(get_model_urls("ipfabric_netbox", "ipfabricbranch"))
+        "sync/<int:pk>/delete/",
+        views.IPFabricSyncDeleteView.as_view(),
+        name="ipfabricsync_delete",
+    ),
+    # Ingestion
+    path(
+        "ingestion/",
+        views.IPFabricIngestionListView.as_view(),
+        name="ipfabricingestion_list",
+    ),
+    path(
+        "ingestion/<int:pk>/",
+        include(get_model_urls("ipfabric_netbox", "ipfabricingestion")),
     ),
     # Transform Map
     path(

ipfabric_netbox/utilities/ipfutils.py

@@ -28,7 +28,7 @@ from .nbutils import order_devices
 from .nbutils import order_members
 
 if TYPE_CHECKING:
-    from ..models import IPFabricBranch
+    from ..models import IPFabricIngestion
 
 logger = logging.getLogger("ipfabric_netbox.utilities.ipf_utils")
 
@@ -128,10 +128,16 @@ class IPFabric(object):
 
 class IPFabricSyncRunner(object):
     def __init__(
-        self, transform_map, sync=None, client: IPFabric = None, settings: dict = None
+        self,
+        transform_map,
+        sync=None,
+        client: IPFabric = None,
+        ingestion=None,
+        settings: dict = None,
     ) -> None:
         self.client = client
         self.settings = settings
+        self.ingestion = ingestion
         self.transform_map = transform_map
         self.sync = sync
         if hasattr(self.sync, "logger"):
@@ -153,17 +159,17 @@ class IPFabricSyncRunner(object):
                 if isinstance(err, SearchError):
                     if self.settings.get(err.model):
                         self.logger.log_failure(
-                            f"Aborting syncing {err.model} instance due to above error, please check your transform maps and/or existing data.",
+                            f"Aborting syncing `{err.model}` instance due to above error, please check your transform maps and/or existing data.",
                             obj=self.sync,
                         )
                     else:
                         self.logger.log_failure(
-                            f"Syncing {err.model} is disabled in settings, but hit above error trying to find the correct item. Please check your transform maps and/or existing data.",
+                            f"Syncing `{err.model}` is disabled in settings, but hit above error trying to find the correct item. Please check your transform maps and/or existing data.",
                             obj=self.sync,
                         )
                 else:
                     self.logger.log_failure(
-                        f"Syncing failed with: {err}. See above error for more details.",
+                        f"Syncing failed with: `{err}`. See above error for more details.",
                         obj=self.sync,
                     )
                 # Make sure the whole sync is failed when we encounter error
@@ -172,6 +178,12 @@ class IPFabricSyncRunner(object):
 
         return wrapper
 
+    def get_db_connection_name(self) -> str:
+        connection_name = None
+        if self.ingestion:
+            connection_name = self.ingestion.branch.connection_name
+        return connection_name
+
     def get_model_or_update(self, app, model, data):
         transform_map = self.transform_map.objects.filter(
             target_model__app_label=app, target_model__model=model
@@ -198,16 +210,18 @@ class IPFabricSyncRunner(object):
 
         object = None
         try:
+            connection_name = self.get_db_connection_name()
             if model_settings:
                 logger.info(f"Creating {model}")
                 object = transform_map.update_or_create_instance(
                     context=context,
                     tags=self.sync.tags.all(),
+                    connection_name=connection_name,
                 )
             else:
                 logger.info(f"Getting {model}")
                 context.pop("defaults", None)
-                object = queryset.get(**context)
+                object = queryset.using(connection_name).get(**context)
         except queryset.model.DoesNotExist as err:
             self.logger.log_failure(
                 f"<b>`{model}` with these keys not found: `{context}`.</b><br/>Original data: `{data}`.",
@@ -474,6 +488,16 @@ class IPFabricSyncRunner(object):
                 else:
                     managed_ips[ip["sn"]][int_name].append(ip)
 
+        for vlan in data["vlan"][:]:
+            # Remove VLANs with ID 0, minimum VLAN ID in NetBox is 1
+            if vlan.get("vlanId") == 0:
+                data["vlan"].remove(vlan)
+
+        for item in data["inventoryitem"][:]:
+            # Remove items with empty serial number
+            if item.get("sn") in [None, "None"]:
+                data["inventoryitem"].remove(item)
+
         for model, item_count in [
             ("site", len(data.get("site", []))),
             ("device", len(devices)),
@@ -513,7 +537,7 @@ class IPFabricSyncRunner(object):
         app_label: str,
         model: str,
         cf: bool = False,
-        branch: "IPFabricBranch" = None,
+        ingestion: "IPFabricIngestion" = None,
     ) -> None:
         """Sync list of items to NetBox."""
         if not self.settings.get(model):
@@ -530,17 +554,18 @@ class IPFabricSyncRunner(object):
             self.logger.increment_statistics(model=model)
 
             if cf:
+                synced_object.snapshot()
                 synced_object.custom_field_data[
                     "ipfabric_source"
                 ] = self.sync.snapshot_data.source.pk
-                if branch:
-                    synced_object.custom_field_data["ipfabric_branch"] = branch.pk
+                if ingestion:
+                    synced_object.custom_field_data["ipfabric_ingestion"] = ingestion.pk
                 synced_object.save()
 
     @handle_errors
     def sync_devices(
         self,
-        branch,
+        ingestion,
         devices,
         interface_dict,
         managed_ips,
@@ -578,11 +603,12 @@ class IPFabricSyncRunner(object):
                 continue
 
             if self.settings.get("device"):
+                device_object.snapshot()
                 device_object.custom_field_data[
                     "ipfabric_source"
                 ] = self.sync.snapshot_data.source.pk
-                if branch:
-                    device_object.custom_field_data["ipfabric_branch"] = branch.pk
+                if ingestion:
+                    device_object.custom_field_data["ipfabric_ingestion"] = ingestion.pk
                 device_object.save()
 
             self.logger.increment_statistics(model="device")
@@ -622,20 +648,26 @@ class IPFabricSyncRunner(object):
             return
         self.logger.increment_statistics(model="ipaddress")
 
+        connection_name = self.get_db_connection_name()
+
         try:
             # Removing other IP is done in .signals.clear_other_primary_ip
             # But do it here too so the change is shown in StagedChange diff
-            other_device = Device.objects.get(primary_ip4=ip_address_obj)
+            other_device = Device.objects.using(connection_name).get(
+                primary_ip4=ip_address_obj
+            )
             if other_device and device_object != other_device:
+                other_device.snapshot()
                 other_device.primary_ip4 = None
-                other_device.save()
+                other_device.save(using=connection_name)
         except ObjectDoesNotExist:
             pass
 
         if login_ip == primary_ip:
             try:
+                device_object.snapshot()
                 device_object.primary_ip4 = ip_address_obj
-                device_object.save()
+                device_object.save(using=connection_name)
             except ValueError as err:
                 self.logger.log_failure(
                     f"Error assigning primary IP to device: {err}", obj=self.sync
@@ -659,8 +691,9 @@ class IPFabricSyncRunner(object):
             "dcim", "macaddress", macaddress_data
         )
         try:
+            interface_object.snapshot()
             interface_object.primary_mac_address = macaddress_object
-            interface_object.save()
+            interface_object.save(using=self.get_db_connection_name())
         except ValueError as err:
             self.logger.log_failure(
                 f"Error assigning MAC Address to interface: {err}", obj=self.sync
@@ -697,7 +730,7 @@ class IPFabricSyncRunner(object):
 
         return interface_object
 
-    def collect_and_sync(self, branch=None) -> None:
+    def collect_and_sync(self, ingestion=None) -> None:
         self.logger.log_info("Starting data sync.", obj=self.sync)
         (
             sites,
@@ -711,10 +744,10 @@ class IPFabricSyncRunner(object):
         ) = self.collect_data()
 
         self.sync_items(
-            app_label="dcim", model="site", items=sites, cf=True, branch=branch
+            app_label="dcim", model="site", items=sites, cf=True, ingestion=ingestion
         )
         self.sync_devices(
-            branch,
+            ingestion,
             devices,
             interface_dict,
             managed_ips,