ipfabric_netbox 4.2.0b4__py3-none-any.whl → 4.2.0b6__py3-none-any.whl

ipfabric_netbox/__init__.py

@@ -6,7 +6,7 @@ class NetboxIPFabricConfig(PluginConfig):
     name = "ipfabric_netbox"
     verbose_name = "NetBox IP Fabric SoT Plugin"
     description = "Sync IP Fabric into NetBox"
-    version = "4.2.0b4"
+    version = "4.2.0b6"
     base_url = "ipfabric"
     min_version = "4.2.4"
 

ipfabric_netbox/api/__init__.py

@@ -1,2 +1 @@
-from .nested_serializers import *  # noqa: F401, F403
 from .serializers import *  # noqa: F401, F403

ipfabric_netbox/api/serializers.py

@@ -1,19 +1,19 @@
 from core.choices import DataSourceStatusChoices
+from django.contrib.contenttypes.models import ContentType
 from netbox.api.fields import ChoiceField
 from netbox.api.fields import ContentTypeField
-from netbox.api.serializers import NetBoxModelSerializer
+from netbox.api.fields import RelatedObjectCountField
+from netbox.api.serializers import NestedGroupModelSerializer
 from netbox_branching.api.serializers import BranchSerializer
 from rest_framework import serializers
 
-from .nested_serializers import NestedIPFabricSnapshotSerializer
-from .nested_serializers import NestedIPFabricSourceSerializer
-from .nested_serializers import NestedIPFabricSyncSerializer
-from .nested_serializers import NestedIPFabricTransformMapSerializer
 from ipfabric_netbox.models import IPFabricIngestion
 from ipfabric_netbox.models import IPFabricIngestionIssue
 from ipfabric_netbox.models import IPFabricRelationshipField
+from ipfabric_netbox.models import IPFabricRelationshipFieldSourceModels
 from ipfabric_netbox.models import IPFabricSnapshot
 from ipfabric_netbox.models import IPFabricSource
+from ipfabric_netbox.models import IPFabricSupportedSyncModels
 from ipfabric_netbox.models import IPFabricSync
 from ipfabric_netbox.models import IPFabricTransformField
 from ipfabric_netbox.models import IPFabricTransformMap
@@ -32,129 +32,202 @@ __all__ = (
 )
 
 
-class IPFabricSyncSerializer(NetBoxModelSerializer):
-    snapshot_data = NestedIPFabricSnapshotSerializer(read_only=True)
+class IPFabricTransformMapGroupSerializer(NestedGroupModelSerializer):
+    transform_maps_count = RelatedObjectCountField("transform_maps")
 
     class Meta:
-        model = IPFabricSync
-        fields = [
+        model = IPFabricTransformMapGroup
+        fields = (
             "id",
             "name",
-            "display",
-            "snapshot_data",
-            "type",
-            "status",
-            "parameters",
-            "last_synced",
+            "description",
+            "transform_maps_count",
             "created",
             "last_updated",
-        ]
+        )
+        brief_fields = (
+            "id",
+            "name",
+            "description",
+        )
 
 
-class IPFabricSnapshotSerializer(NetBoxModelSerializer):
-    source = NestedIPFabricSourceSerializer()
-    data = serializers.JSONField()
-    date = serializers.DateTimeField()
-    last_updated = serializers.DateTimeField()
+class IPFabricTransformMapSerializer(NestedGroupModelSerializer):
+    group = IPFabricTransformMapGroupSerializer(
+        nested=True, required=False, allow_null=True
+    )
+    target_model = ContentTypeField(
+        queryset=ContentType.objects.filter(IPFabricSupportedSyncModels)
+    )
 
     class Meta:
-        model = IPFabricSnapshot
-        fields = [
+        model = IPFabricTransformMap
+        fields = (
             "id",
             "name",
-            "source",
-            "snapshot_id",
-            "status",
-            "date",
-            "display",
-            "sites",
-            "data",
+            "group",
+            "source_model",
+            "target_model",
             "created",
             "last_updated",
-        ]
+        )
+        brief_fields = (
+            "id",
+            "name",
+            "group",
+            "source_model",
+            "target_model",
+        )
 
-        extra_kwargs = {
-            "raw_data": {"write_only": True},
-        }
 
+class IPFabricTransformFieldSerializer(NestedGroupModelSerializer):
+    transform_map = IPFabricTransformMapSerializer(nested=True)
+
+    class Meta:
+        model = IPFabricTransformField
+        fields = (
+            "id",
+            "transform_map",
+            "source_field",
+            "target_field",
+            "coalesce",
+            "template",
+        )
 
-class IPFabricRelationshipFieldSerializer(NetBoxModelSerializer):
-    transform_map = NestedIPFabricTransformMapSerializer(read_only=True)
-    source_model = ContentTypeField(read_only=True)
+
+class IPFabricRelationshipFieldSerializer(NestedGroupModelSerializer):
+    transform_map = IPFabricTransformMapSerializer(nested=True)
+    source_model = ContentTypeField(
+        queryset=ContentType.objects.filter(IPFabricRelationshipFieldSourceModels)
+    )
 
     class Meta:
         model = IPFabricRelationshipField
-        fields = [
+        fields = (
             "id",
             "transform_map",
             "source_model",
             "target_field",
             "coalesce",
             "template",
-        ]
+        )
 
 
-class IPFabricTransformMapGroupSerializer(NetBoxModelSerializer):
+class IPFabricSourceSerializer(NestedGroupModelSerializer):
+    status = ChoiceField(choices=DataSourceStatusChoices, read_only=True)
+    url = serializers.URLField()
+
     class Meta:
-        model = IPFabricTransformMapGroup
-        fields = [
+        model = IPFabricSource
+        fields = (
+            "id",
+            "url",
+            "display",
             "name",
+            "type",
+            "status",
+            "last_synced",
             "description",
-            "transform_maps",
+            "comments",
+            "parameters",
             "created",
             "last_updated",
-        ]
+        )
+        brief_fields = (
+            "display",
+            "id",
+            "name",
+            "status",
+            "type",
+            "url",
+        )
 
 
-class IPFabricTransformMapSerializer(NetBoxModelSerializer):
-    target_model = ContentTypeField(read_only=True)
+class IPFabricSnapshotSerializer(NestedGroupModelSerializer):
+    source = IPFabricSourceSerializer(nested=True, read_only=True)
+    data = serializers.JSONField()
+    display = serializers.CharField(source="__str__", read_only=True)
 
     class Meta:
-        model = IPFabricTransformMap
-        fields = [
+        model = IPFabricSnapshot
+        fields = (
             "id",
-            "source_model",
-            "target_model",
+            "display",
+            "name",
+            "source",
+            "snapshot_id",
+            "status",
+            "data",
+            "date",
             "created",
             "last_updated",
-        ]
+        )
+        brief_fields = (
+            "display",
+            "id",
+            "name",
+            "source",
+            "snapshot_id",
+            "status",
+            "data",
+            "date",
+        )
 
 
-class IPFabricTransformFieldSerializer(NetBoxModelSerializer):
-    transform_map = NestedIPFabricTransformMapSerializer(read_only=True)
+class IPFabricSyncSerializer(NestedGroupModelSerializer):
+    snapshot_data = IPFabricSnapshotSerializer(nested=True)
 
     class Meta:
-        model = IPFabricTransformField
-        fields = [
+        model = IPFabricSync
+        fields = (
             "id",
-            "transform_map",
-            "source_field",
-            "target_field",
-            "coalesce",
-            "template",
-        ]
+            "name",
+            "snapshot_data",
+            "type",
+            "status",
+            "parameters",
+            "auto_merge",
+            "last_synced",
+            "scheduled",
+            "interval",
+            "user",
+        )
+        brief_fields = (
+            "auto_merge",
+            "id",
+            "last_synced",
+            "name",
+            "parameters",
+            "status",
+        )
 
 
-class IPFabricIngestionSerializer(NetBoxModelSerializer):
+class IPFabricIngestionSerializer(NestedGroupModelSerializer):
     branch = BranchSerializer(read_only=True)
-    sync = NestedIPFabricSyncSerializer(read_only=True)
+    sync = IPFabricSyncSerializer(nested=True)
 
     class Meta:
         model = IPFabricIngestion
-        fields = [
+        fields = (
+            "id",
+            "name",
+            "branch",
+            "sync",
+        )
+        brief_fields = (
             "id",
             "name",
             "branch",
             "sync",
-        ]
+        )
 
 
-class IPFabricIngestionIssueSerializer(NetBoxModelSerializer):
-    ingestion = IPFabricIngestionSerializer(read_only=True)
+class IPFabricIngestionIssueSerializer(NestedGroupModelSerializer):
+    ingestion = IPFabricIngestionSerializer(nested=True)
 
     class Meta:
         model = IPFabricIngestionIssue
-        fields = [
+        fields = (
             "id",
             "ingestion",
             "timestamp",
@@ -164,26 +237,11 @@ class IPFabricIngestionIssueSerializer(NetBoxModelSerializer):
             "coalesce_fields",
             "defaults",
             "exception",
-        ]
-
-
-class IPFabricSourceSerializer(NetBoxModelSerializer):
-    status = ChoiceField(choices=DataSourceStatusChoices)
-    url = serializers.URLField()
-
-    class Meta:
-        model = IPFabricSource
-        fields = [
+        )
+        brief_fields = (
+            "exception",
             "id",
-            "url",
-            "display",
-            "name",
-            "type",
-            "status",
-            "last_synced",
-            "description",
-            "comments",
-            "parameters",
-            "created",
-            "last_updated",
-        ]
+            "ingestion",
+            "message",
+            "model",
+        )

ipfabric_netbox/api/urls.py

@@ -3,11 +3,11 @@ from netbox.api.routers import NetBoxRouter
 
 from ipfabric_netbox.api.views import IPFabricIngestionIssueViewSet
 from ipfabric_netbox.api.views import IPFabricIngestionViewSet
-from ipfabric_netbox.api.views import IPFabricRelationshipFieldiewSet
+from ipfabric_netbox.api.views import IPFabricRelationshipFieldViewSet
 from ipfabric_netbox.api.views import IPFabricSnapshotViewSet
 from ipfabric_netbox.api.views import IPFabricSourceViewSet
 from ipfabric_netbox.api.views import IPFabricSyncViewSet
-from ipfabric_netbox.api.views import IPFabricTransformFieldiewSet
+from ipfabric_netbox.api.views import IPFabricTransformFieldViewSet
 from ipfabric_netbox.api.views import IPFabricTransformMapGroupViewSet
 from ipfabric_netbox.api.views import IPFabricTransformMapViewSet
 
@@ -20,6 +20,6 @@ router.register("transform-map", IPFabricTransformMapViewSet)
 router.register("sync", IPFabricSyncViewSet)
 router.register("ingestion", IPFabricIngestionViewSet)
 router.register("ingestion-issues", IPFabricIngestionIssueViewSet)
-router.register("transform-field", IPFabricTransformFieldiewSet)
-router.register("relationship-field", IPFabricRelationshipFieldiewSet)
+router.register("transform-field", IPFabricTransformFieldViewSet)
+router.register("relationship-field", IPFabricRelationshipFieldViewSet)
 urlpatterns = router.urls

ipfabric_netbox/api/views.py

@@ -14,6 +14,7 @@ from .serializers import IPFabricSyncSerializer
 from .serializers import IPFabricTransformFieldSerializer
 from .serializers import IPFabricTransformMapGroupSerializer
 from .serializers import IPFabricTransformMapSerializer
+from ipfabric_netbox.filtersets import IPFabricRelationshipFieldFilterSet
 from ipfabric_netbox.filtersets import IPFabricSnapshotFilterSet
 from ipfabric_netbox.filtersets import IPFabricSourceFilterSet
 from ipfabric_netbox.filtersets import IPFabricTransformFieldFilterSet
@@ -29,29 +30,29 @@ from ipfabric_netbox.models import IPFabricTransformMap
 from ipfabric_netbox.models import IPFabricTransformMapGroup
 
 
-class IPFabricTransformMapGroupViewSet(NetBoxReadOnlyModelViewSet):
+class IPFabricTransformMapGroupViewSet(NetBoxModelViewSet):
     queryset = IPFabricTransformMapGroup.objects.all()
     serializer_class = IPFabricTransformMapGroupSerializer
 
 
-class IPFabricTransformMapViewSet(NetBoxReadOnlyModelViewSet):
+class IPFabricTransformMapViewSet(NetBoxModelViewSet):
     queryset = IPFabricTransformMap.objects.all()
     serializer_class = IPFabricTransformMapSerializer
 
 
-class IPFabricTransformFieldiewSet(NetBoxReadOnlyModelViewSet):
+class IPFabricTransformFieldViewSet(NetBoxModelViewSet):
     queryset = IPFabricTransformField.objects.all()
     serializer_class = IPFabricTransformFieldSerializer
     filterset_class = IPFabricTransformFieldFilterSet
 
 
-class IPFabricRelationshipFieldiewSet(NetBoxReadOnlyModelViewSet):
+class IPFabricRelationshipFieldViewSet(NetBoxModelViewSet):
     queryset = IPFabricRelationshipField.objects.all()
     serializer_class = IPFabricRelationshipFieldSerializer
-    filterset_class = IPFabricTransformFieldFilterSet
+    filterset_class = IPFabricRelationshipFieldFilterSet
 
 
-class IPFabricSyncViewSet(NetBoxReadOnlyModelViewSet):
+class IPFabricSyncViewSet(NetBoxModelViewSet):
     queryset = IPFabricSync.objects.all()
     serializer_class = IPFabricSyncSerializer
 
@@ -66,7 +67,7 @@ class IPFabricIngestionIssueViewSet(NetBoxReadOnlyModelViewSet):
     serializer_class = IPFabricIngestionIssueSerializer
 
 
-class IPFabricSnapshotViewSet(NetBoxModelViewSet):
+class IPFabricSnapshotViewSet(NetBoxReadOnlyModelViewSet):
     queryset = IPFabricSnapshot.objects.all()
     serializer_class = IPFabricSnapshotSerializer
     filterset_class = IPFabricSnapshotFilterSet
@@ -79,24 +80,23 @@ class IPFabricSnapshotViewSet(NetBoxModelViewSet):
             raw_data._raw_delete(raw_data.db)
             return Response({"status": "success"})
         elif request.method == "PATCH":
-            transaction.set_autocommit(False)
-            IPFabricData.objects.bulk_create(
-                [
-                    IPFabricData(
-                        snapshot_data=snapshot, data=item["data"], type=item["type"]
-                    )
-                    for item in request.data["data"]
-                ],
-                batch_size=5000,
-            )
-            transaction.commit()
+            with transaction.atomic():
+                IPFabricData.objects.bulk_create(
+                    [
+                        IPFabricData(
+                            snapshot_data=snapshot, data=item["data"], type=item["type"]
+                        )
+                        for item in request.data["data"]
+                    ],
+                    batch_size=5000,
+                )
             return Response({"status": "success"})
 
     @action(detail=True, methods=["get"], url_path="sites")
     def sites(self, request, pk):
         q = request.GET.get("q", None)
         snapshot = IPFabricSnapshot.objects.get(pk=pk)
-        new_sites = {"results": []}
+        new_sites = {"count": 0, "results": []}
         if snapshot.data:
             sites = snapshot.data.get("sites", None)
             num = 0
@@ -112,6 +112,7 @@ class IPFabricSnapshotViewSet(NetBoxModelViewSet):
                             {"display": site, "name": site, "id": site}
                         )
                     num += 1
+                new_sites["count"] = num
                 return Response(new_sites)
         else:
             return Response([])

ipfabric_netbox/filtersets.py

@@ -11,9 +11,11 @@ from netbox_branching.models import ChangeDiff
 from .models import IPFabricData
 from .models import IPFabricIngestion
 from .models import IPFabricIngestionIssue
+from .models import IPFabricRelationshipField
 from .models import IPFabricSnapshot
 from .models import IPFabricSource
 from .models import IPFabricSync
+from .models import IPFabricTransformField
 from .models import IPFabricTransformMap
 from .models import IPFabricTransformMapGroup
 
@@ -193,3 +195,67 @@ class IPFabricTransformFieldFilterSet(BaseFilterSet):
     transform_map = django_filters.ModelMultipleChoiceFilter(
         queryset=IPFabricTransformMap.objects.all(), label=_("Transform Map")
     )
+
+    class Meta:
+        model = IPFabricTransformField
+        fields = (
+            "id",
+            "transform_map",
+            "source_field",
+            "target_field",
+            "coalesce",
+            "template",
+        )
+
+
+class IPFabricRelationshipFieldFilterSet(BaseFilterSet):
+    transform_map = django_filters.ModelMultipleChoiceFilter(
+        queryset=IPFabricTransformMap.objects.all(), label=_("Transform Map")
+    )
+
+    class Meta:
+        model = IPFabricRelationshipField
+        fields = (
+            "id",
+            "transform_map",
+            "source_model",
+            "target_field",
+            "coalesce",
+            "template",
+        )
+
+
+class IPFabricSyncFilterSet(ChangeLoggedModelFilterSet):
+    q = django_filters.CharFilter(method="search")
+    snapshot_data_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=IPFabricSnapshot.objects.all(),
+        label=_("Snapshot (ID)"),
+    )
+    snapshot_data = django_filters.ModelMultipleChoiceFilter(
+        field_name="snapshot_data__name",
+        queryset=IPFabricSnapshot.objects.all(),
+        to_field_name="name",
+        label=_("Snapshot (name)"),
+    )
+
+    class Meta:
+        model = IPFabricSync
+        fields = (
+            "id",
+            "name",
+            "snapshot_data",
+            "snapshot_data_id",
+            "type",
+            "status",
+            "auto_merge",
+            "last_synced",
+            "scheduled",
+            "interval",
+        )
+
+    def search(self, queryset, name, value):
+        if not value.strip():
+            return queryset
+        return queryset.filter(
+            Q(name__icontains=value) | Q(snapshot_data__name__icontains=value)
+        )

ipfabric_netbox/forms.py

@@ -493,6 +493,12 @@ class IPFabricSyncForm(NetBoxModelForm):
         ),
     )
 
+    update_custom_fields = forms.BooleanField(
+        required=False,
+        label=_("Custom Fields Updating"),
+        help_text=_("Update object custom fields where applicable."),
+    )
+
     scheduled = forms.DateTimeField(
         required=False,
         widget=DateTimePicker(),
@@ -519,6 +525,7 @@ class IPFabricSyncForm(NetBoxModelForm):
             "source",
             "snapshot_data",
             "auto_merge",
+            "update_custom_fields",
             "sites",
             "type",
             "tags",
@@ -550,7 +557,9 @@ class IPFabricSyncForm(NetBoxModelForm):
         fieldsets.append(
             FieldSet("scheduled", "interval", name=_("Ingestion Execution Parameters"))
         )
-        fieldsets.append(FieldSet("auto_merge", name=_("Extras")))
+        fieldsets.append(
+            FieldSet("auto_merge", "update_custom_fields", name=_("Extras"))
+        )
         fieldsets.append(FieldSet("tags", name=_("Tags")))
 
         return fieldsets

ipfabric_netbox/graphql/__init__.py

@@ -0,0 +1,23 @@
+from .schema import IPFabricDataQuery
+from .schema import IPFabricIngestionIssueQuery
+from .schema import IPFabricIngestionQuery
+from .schema import IPFabricRelationshipFieldQuery
+from .schema import IPFabricSnapshotQuery
+from .schema import IPFabricSourceQuery
+from .schema import IPFabricSyncQuery
+from .schema import IPFabricTransformFieldQuery
+from .schema import IPFabricTransformMapGroupQuery
+from .schema import IPFabricTransformMapQuery
+
+schema = [
+    IPFabricTransformMapGroupQuery,
+    IPFabricTransformMapQuery,
+    IPFabricTransformFieldQuery,
+    IPFabricRelationshipFieldQuery,
+    IPFabricSourceQuery,
+    IPFabricSnapshotQuery,
+    IPFabricSyncQuery,
+    IPFabricIngestionQuery,
+    IPFabricIngestionIssueQuery,
+    IPFabricDataQuery,
+]

ipfabric_netbox/graphql/enums.py

@@ -0,0 +1,38 @@
+import strawberry
+from core.choices import DataSourceStatusChoices
+from core.choices import JobStatusChoices
+from netbox_branching.choices import BranchStatusChoices
+
+from ipfabric_netbox.choices import IPFabricRawDataTypeChoices
+from ipfabric_netbox.choices import IPFabricSnapshotStatusModelChoices
+from ipfabric_netbox.choices import IPFabricSourceTypeChoices
+from ipfabric_netbox.choices import IPFabricSyncTypeChoices
+from ipfabric_netbox.choices import IPFabricTransformMapSourceModelChoices
+
+__all__ = (
+    "DataSourceStatusEnum",
+    "IPFabricTransformMapSourceModelEnum",
+    "IPFabricSourceTypeEnum",
+    "IPFabricSnapshotStatusModelEnum",
+    "IPFabricSyncTypeEnum",
+    "IPFabricRawDataTypeEnum",
+    "BranchStatusEnum",
+    "JobStatusEnum",
+)
+
+DataSourceStatusEnum = strawberry.enum(DataSourceStatusChoices.as_enum(prefix="type"))
+IPFabricTransformMapSourceModelEnum = strawberry.enum(
+    IPFabricTransformMapSourceModelChoices.as_enum(prefix="type")
+)
+IPFabricSourceTypeEnum = strawberry.enum(
+    IPFabricSourceTypeChoices.as_enum(prefix="type")
+)
+IPFabricSnapshotStatusModelEnum = strawberry.enum(
+    IPFabricSnapshotStatusModelChoices.as_enum(prefix="type")
+)
+IPFabricSyncTypeEnum = strawberry.enum(IPFabricSyncTypeChoices.as_enum(prefix="type"))
+IPFabricRawDataTypeEnum = strawberry.enum(
+    IPFabricRawDataTypeChoices.as_enum(prefix="type")
+)
+BranchStatusEnum = strawberry.enum(BranchStatusChoices.as_enum(prefix="type"))
+JobStatusEnum = strawberry.enum(JobStatusChoices.as_enum(prefix="type"))