ipfabric_netbox 4.3.2b10__py3-none-any.whl → 4.3.2b11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ipfabric_netbox might be problematic. Click here for more details.
- ipfabric_netbox/__init__.py +2 -2
- ipfabric_netbox/choices.py +2 -0
- ipfabric_netbox/data/endpoint.json +5 -0
- ipfabric_netbox/data/filters.json +1 -1
- ipfabric_netbox/data/transform_map.json +3 -3
- ipfabric_netbox/forms.py +10 -5
- ipfabric_netbox/jobs.py +10 -3
- ipfabric_netbox/migrations/0023_populate_filters_data.py +24 -0
- ipfabric_netbox/migrations/0025_add_vss_chassis_endpoint.py +166 -0
- ipfabric_netbox/models.py +72 -29
- ipfabric_netbox/templatetags/ipfabric_netbox_helpers.py +7 -4
- ipfabric_netbox/tests/test_forms.py +93 -0
- ipfabric_netbox/tests/test_views.py +1 -1
- ipfabric_netbox/utilities/endpoint.py +53 -0
- ipfabric_netbox/utilities/ipfutils.py +252 -174
- ipfabric_netbox/utilities/transform_map.py +18 -5
- {ipfabric_netbox-4.3.2b10.dist-info → ipfabric_netbox-4.3.2b11.dist-info}/METADATA +3 -4
- {ipfabric_netbox-4.3.2b10.dist-info → ipfabric_netbox-4.3.2b11.dist-info}/RECORD +19 -18
- {ipfabric_netbox-4.3.2b10.dist-info → ipfabric_netbox-4.3.2b11.dist-info}/WHEEL +1 -1
ipfabric_netbox/__init__.py
CHANGED
|
@@ -6,9 +6,9 @@ class NetboxIPFabricConfig(PluginConfig):
|
|
|
6
6
|
name = "ipfabric_netbox"
|
|
7
7
|
verbose_name = "NetBox IP Fabric SoT Plugin"
|
|
8
8
|
description = "Sync IP Fabric into NetBox"
|
|
9
|
-
version = "4.3.
|
|
9
|
+
version = "4.3.2b11"
|
|
10
10
|
base_url = "ipfabric"
|
|
11
|
-
min_version = "4.4.
|
|
11
|
+
min_version = "4.4.0"
|
|
12
12
|
|
|
13
13
|
def ready(self):
|
|
14
14
|
super().ready()
|
ipfabric_netbox/choices.py
CHANGED
|
@@ -173,6 +173,7 @@ class IPFabricEndpointChoices(ChoiceSet):
|
|
|
173
173
|
SITES = "/inventory/sites/overview"
|
|
174
174
|
DEVICES = "/inventory/devices"
|
|
175
175
|
VIRTUALCHASSIS = "/technology/platforms/stack/members"
|
|
176
|
+
VSS_CHASSIS = "/technology/platforms/vss/chassis"
|
|
176
177
|
INTERFACES = "/inventory/interfaces"
|
|
177
178
|
PARTNUMBERS = "/inventory/part-numbers"
|
|
178
179
|
VLANS = "/technology/vlans/site-summary"
|
|
@@ -184,6 +185,7 @@ class IPFabricEndpointChoices(ChoiceSet):
|
|
|
184
185
|
(SITES, SITES, "cyan"),
|
|
185
186
|
(DEVICES, DEVICES, "gray"),
|
|
186
187
|
(VIRTUALCHASSIS, VIRTUALCHASSIS, "grey"),
|
|
188
|
+
(VSS_CHASSIS, VSS_CHASSIS, "gray"),
|
|
187
189
|
(INTERFACES, INTERFACES, "gray"),
|
|
188
190
|
(PARTNUMBERS, PARTNUMBERS, "gray"),
|
|
189
191
|
(VLANS, VLANS, "gray"),
|
|
@@ -14,6 +14,11 @@
|
|
|
14
14
|
"description": "",
|
|
15
15
|
"endpoint": "/technology/platforms/stack/members"
|
|
16
16
|
},
|
|
17
|
+
{
|
|
18
|
+
"name": "Default VSS Chassis Endpoint",
|
|
19
|
+
"description": "",
|
|
20
|
+
"endpoint": "/technology/platforms/vss/chassis"
|
|
21
|
+
},
|
|
17
22
|
{
|
|
18
23
|
"name": "Default Interfaces Endpoint",
|
|
19
24
|
"description": "",
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
},
|
|
30
30
|
{
|
|
31
31
|
"name": "Default Child Devices Filter",
|
|
32
|
-
"endpoints": ["/technology/platforms/stack/members", "/inventory/interfaces", "/technology/routing/vrf/detail", "/technology/addressing/managed-ip/ipv4"],
|
|
32
|
+
"endpoints": ["/technology/platforms/stack/members", "/technology/platforms/vss/chassis", "/inventory/interfaces", "/technology/routing/vrf/detail", "/technology/addressing/managed-ip/ipv4"],
|
|
33
33
|
"expressions": ["Default Device Child Filter Expression"]
|
|
34
34
|
},
|
|
35
35
|
{
|
|
@@ -153,7 +153,7 @@
|
|
|
153
153
|
"source_field": "master",
|
|
154
154
|
"target_field": "name",
|
|
155
155
|
"coalesce": true,
|
|
156
|
-
"template": ""
|
|
156
|
+
"template": "{% if object.master is defined and object.master %}{{ object.master }}{% else %}{{ object.hostname }}{% endif %}"
|
|
157
157
|
}
|
|
158
158
|
],
|
|
159
159
|
"relationship_maps": [
|
|
@@ -233,7 +233,7 @@
|
|
|
233
233
|
"source_field": "hostname",
|
|
234
234
|
"target_field": "vc_position",
|
|
235
235
|
"coalesce": false,
|
|
236
|
-
"template": "{% if object.virtual_chassis %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}"
|
|
236
|
+
"template": "{% if object.virtual_chassis %}{% if object.virtual_chassis.member is defined and object.virtual_chassis.member %}{{ object.virtual_chassis.member }}{% else %}{{ object.virtual_chassis.chassisId }}{% endif %}{% else %}None{% endif %}"
|
|
237
237
|
}
|
|
238
238
|
],
|
|
239
239
|
"relationship_maps": [
|
|
@@ -244,7 +244,7 @@
|
|
|
244
244
|
},
|
|
245
245
|
"target_field": "virtual_chassis",
|
|
246
246
|
"coalesce": false,
|
|
247
|
-
"template": "{% if object.virtual_chassis %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}"
|
|
247
|
+
"template": "{% if object.virtual_chassis %}{% if object.virtual_chassis.master is defined and object.virtual_chassis.master %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% else %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.hostname).first().pk }}{% endif %}{% endif %}"
|
|
248
248
|
},
|
|
249
249
|
{
|
|
250
250
|
"source_model": {
|
ipfabric_netbox/forms.py
CHANGED
|
@@ -630,10 +630,10 @@ class IPFabricSyncForm(NetBoxModelForm):
|
|
|
630
630
|
),
|
|
631
631
|
)
|
|
632
632
|
|
|
633
|
-
filters =
|
|
633
|
+
filters = DynamicModelMultipleChoiceField(
|
|
634
634
|
queryset=IPFabricFilter.objects.all(),
|
|
635
|
+
label=_("Filters"),
|
|
635
636
|
required=False,
|
|
636
|
-
widget=forms.SelectMultiple(),
|
|
637
637
|
)
|
|
638
638
|
|
|
639
639
|
update_custom_fields = forms.BooleanField(
|
|
@@ -773,8 +773,8 @@ class IPFabricSyncForm(NetBoxModelForm):
|
|
|
773
773
|
|
|
774
774
|
# Prepare buttons for each target Model, order according to model hierarchy
|
|
775
775
|
hierarchy = [
|
|
776
|
-
f"{
|
|
777
|
-
for
|
|
776
|
+
f"{tm.target_model.app_label}.{tm.target_model.model}"
|
|
777
|
+
for tm in IPFabricSync.get_model_hierarchy(
|
|
778
778
|
group_ids=self.initial.get("groups", [])
|
|
779
779
|
)
|
|
780
780
|
]
|
|
@@ -903,7 +903,12 @@ class IPFabricSyncForm(NetBoxModelForm):
|
|
|
903
903
|
]
|
|
904
904
|
self.instance.parameters = dict(sorted(parameters.items()))
|
|
905
905
|
self.instance.status = IPFabricSyncStatusChoices.NEW
|
|
906
|
-
|
|
906
|
+
|
|
907
|
+
instance = super().save(*args, **kwargs)
|
|
908
|
+
# M2M relationships need to be set after the instance is saved
|
|
909
|
+
# But only if they are set from the side where they are not defined on model
|
|
910
|
+
instance.filters.set(self.cleaned_data["filters"])
|
|
911
|
+
return instance
|
|
907
912
|
|
|
908
913
|
|
|
909
914
|
class IPFabricSyncBulkEditForm(NetBoxModelBulkEditForm):
|
ipfabric_netbox/jobs.py
CHANGED
|
@@ -6,7 +6,12 @@ from core.exceptions import SyncError
|
|
|
6
6
|
from dcim.models import Site
|
|
7
7
|
from dcim.models import VirtualChassis
|
|
8
8
|
from dcim.signals import assign_virtualchassis_master
|
|
9
|
-
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
# Got added in NetBox 4.4.9
|
|
12
|
+
from dcim.signals import sync_cached_scope_fields
|
|
13
|
+
except ImportError:
|
|
14
|
+
sync_cached_scope_fields = None
|
|
10
15
|
from django.db.models import signals
|
|
11
16
|
from netbox.context_managers import event_tracking
|
|
12
17
|
from rq.timeouts import JobTimeoutException
|
|
@@ -126,14 +131,16 @@ def merge_ipfabric_ingestion(job, remove_branch=False, *args, **kwargs):
|
|
|
126
131
|
signals.post_save.disconnect(
|
|
127
132
|
assign_virtualchassis_master, sender=VirtualChassis
|
|
128
133
|
)
|
|
129
|
-
|
|
134
|
+
if sync_cached_scope_fields is not None:
|
|
135
|
+
signals.post_save.disconnect(sync_cached_scope_fields, sender=Site)
|
|
130
136
|
ingestion.sync_merge()
|
|
131
137
|
finally:
|
|
132
138
|
# Re-enable the disabled signals
|
|
133
139
|
signals.post_save.connect(
|
|
134
140
|
assign_virtualchassis_master, sender=VirtualChassis
|
|
135
141
|
)
|
|
136
|
-
|
|
142
|
+
if sync_cached_scope_fields is not None:
|
|
143
|
+
signals.post_save.connect(sync_cached_scope_fields, sender=Site)
|
|
137
144
|
if remove_branch:
|
|
138
145
|
branching_branch = ingestion.branch
|
|
139
146
|
ingestion.branch = None
|
|
@@ -45,6 +45,7 @@ def migrate_source_model_to_endpoint(
|
|
|
45
45
|
"""Migrate IPFabricTransformMap source_model data to source_endpoint."""
|
|
46
46
|
IPFabricTransformMap = apps.get_model("ipfabric_netbox", "IPFabricTransformMap")
|
|
47
47
|
IPFabricEndpoint = apps.get_model("ipfabric_netbox", "IPFabricEndpoint")
|
|
48
|
+
ContentType = apps.get_model("contenttypes", "ContentType")
|
|
48
49
|
|
|
49
50
|
source_model_to_endpoint = {
|
|
50
51
|
"site": "/inventory/sites/overview",
|
|
@@ -69,6 +70,15 @@ def migrate_source_model_to_endpoint(
|
|
|
69
70
|
).all():
|
|
70
71
|
endpoint_value = source_model_to_endpoint.get(transform_map.source_model)
|
|
71
72
|
|
|
73
|
+
# Special case: source_model="device" with target_model="dcim.virtualchassis"
|
|
74
|
+
# should map to /technology/platforms/stack/members, not /inventory/devices
|
|
75
|
+
if transform_map.source_model == "device" and transform_map.target_model:
|
|
76
|
+
virtualchassis_ct = ContentType.objects.using(
|
|
77
|
+
schema_editor.connection.alias
|
|
78
|
+
).get(app_label="dcim", model="virtualchassis")
|
|
79
|
+
if transform_map.target_model_id == virtualchassis_ct.id:
|
|
80
|
+
endpoint_value = "/technology/platforms/stack/members"
|
|
81
|
+
|
|
72
82
|
# If no mapping exists, use fallback and mark the name for manual fix
|
|
73
83
|
if not endpoint_value:
|
|
74
84
|
if not transform_map.name.startswith("[NEEDS CORRECTION]"):
|
|
@@ -97,6 +107,7 @@ def migrate_endpoint_to_source_model(
|
|
|
97
107
|
"""Reverse migration: migrate IPFabricTransformMap source_endpoint data back to source_model."""
|
|
98
108
|
IPFabricTransformMap = apps.get_model("ipfabric_netbox", "IPFabricTransformMap")
|
|
99
109
|
IPFabricEndpoint = apps.get_model("ipfabric_netbox", "IPFabricEndpoint")
|
|
110
|
+
ContentType = apps.get_model("contenttypes", "ContentType")
|
|
100
111
|
|
|
101
112
|
endpoint_to_source_model = {
|
|
102
113
|
"/inventory/sites/overview": "site",
|
|
@@ -122,6 +133,19 @@ def migrate_endpoint_to_source_model(
|
|
|
122
133
|
schema_editor.connection.alias
|
|
123
134
|
).get(pk=transform_map.source_endpoint_id)
|
|
124
135
|
source_model_value = endpoint_to_source_model.get(endpoint.endpoint)
|
|
136
|
+
|
|
137
|
+
# Special case: source_endpoint="/technology/platforms/stack/members" with
|
|
138
|
+
# target_model="dcim.virtualchassis" should map back to source_model="device"
|
|
139
|
+
if (
|
|
140
|
+
endpoint.endpoint == "/technology/platforms/stack/members"
|
|
141
|
+
and transform_map.target_model
|
|
142
|
+
):
|
|
143
|
+
virtualchassis_ct = ContentType.objects.using(
|
|
144
|
+
schema_editor.connection.alias
|
|
145
|
+
).get(app_label="dcim", model="virtualchassis")
|
|
146
|
+
if transform_map.target_model_id == virtualchassis_ct.id:
|
|
147
|
+
source_model_value = "device"
|
|
148
|
+
|
|
125
149
|
transform_map.source_model = source_model_value
|
|
126
150
|
except IPFabricEndpoint.DoesNotExist:
|
|
127
151
|
pass
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING
|
|
2
|
+
|
|
3
|
+
from django.db import migrations
|
|
4
|
+
|
|
5
|
+
from ipfabric_netbox.utilities.endpoint import do_endpoint_change
|
|
6
|
+
from ipfabric_netbox.utilities.endpoint import EndpointRecord
|
|
7
|
+
from ipfabric_netbox.utilities.transform_map import do_change
|
|
8
|
+
from ipfabric_netbox.utilities.transform_map import FieldRecord
|
|
9
|
+
from ipfabric_netbox.utilities.transform_map import RelationshipRecord
|
|
10
|
+
from ipfabric_netbox.utilities.transform_map import TransformMapRecord
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from django.apps import apps as apps_type
|
|
14
|
+
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
|
|
15
|
+
|
|
16
|
+
ENDPOINTS = (
|
|
17
|
+
EndpointRecord(
|
|
18
|
+
name="Default VSS Chassis Endpoint",
|
|
19
|
+
description="",
|
|
20
|
+
endpoint="/technology/platforms/vss/chassis",
|
|
21
|
+
),
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
TRANSFORM_MAP_CHANGES = (
|
|
25
|
+
TransformMapRecord(
|
|
26
|
+
source_endpoint="/technology/platforms/stack/members",
|
|
27
|
+
target_model="dcim.virtualchassis",
|
|
28
|
+
fields=(
|
|
29
|
+
FieldRecord(
|
|
30
|
+
source_field="master",
|
|
31
|
+
target_field="name",
|
|
32
|
+
old_template="",
|
|
33
|
+
new_template="{% if object.master is defined and object.master %}{{ object.master }}{% else %}{{ object.hostname }}{% endif %}",
|
|
34
|
+
),
|
|
35
|
+
),
|
|
36
|
+
),
|
|
37
|
+
TransformMapRecord(
|
|
38
|
+
source_endpoint="/inventory/devices",
|
|
39
|
+
target_model="dcim.device",
|
|
40
|
+
fields=(
|
|
41
|
+
FieldRecord(
|
|
42
|
+
source_field="hostname",
|
|
43
|
+
target_field="vc_position",
|
|
44
|
+
old_template="{% if object.virtual_chassis %}{{ object.virtual_chassis.member }}{% else %}None{% endif %}",
|
|
45
|
+
new_template="{% if object.virtual_chassis %}{% if object.virtual_chassis.member is defined and object.virtual_chassis.member %}{{ object.virtual_chassis.member }}{% else %}{{ object.virtual_chassis.chassisId }}{% endif %}{% else %}None{% endif %}",
|
|
46
|
+
),
|
|
47
|
+
),
|
|
48
|
+
relationships=(
|
|
49
|
+
RelationshipRecord(
|
|
50
|
+
source_model="dcim.virtualchassis",
|
|
51
|
+
target_field="virtual_chassis",
|
|
52
|
+
old_template="{% if object.virtual_chassis %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% endif %}",
|
|
53
|
+
new_template="{% if object.virtual_chassis %}{% if object.virtual_chassis.master is defined and object.virtual_chassis.master %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.master).first().pk }}{% else %}{{ dcim.VirtualChassis.objects.filter(name=object.virtual_chassis.hostname).first().pk }}{% endif %}{% endif %}",
|
|
54
|
+
),
|
|
55
|
+
),
|
|
56
|
+
),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def add_vss_chassis_endpoint(
|
|
61
|
+
apps: "apps_type", schema_editor: "BaseDatabaseSchemaEditor"
|
|
62
|
+
):
|
|
63
|
+
"""Add VSS chassis endpoint."""
|
|
64
|
+
do_endpoint_change(apps, schema_editor, ENDPOINTS, forward=True)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def remove_vss_chassis_endpoint(
|
|
68
|
+
apps: "apps_type", schema_editor: "BaseDatabaseSchemaEditor"
|
|
69
|
+
):
|
|
70
|
+
"""Remove VSS chassis endpoint."""
|
|
71
|
+
do_endpoint_change(apps, schema_editor, ENDPOINTS, forward=False)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def forward_transform_maps_change(
|
|
75
|
+
apps: "apps_type", schema_editor: "BaseDatabaseSchemaEditor"
|
|
76
|
+
):
|
|
77
|
+
"""Replace old template with updated version."""
|
|
78
|
+
do_change(apps, schema_editor, changes=TRANSFORM_MAP_CHANGES, forward=True)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def revert_transform_maps_change(
|
|
82
|
+
apps: "apps_type", schema_editor: "BaseDatabaseSchemaEditor"
|
|
83
|
+
):
|
|
84
|
+
"""Revert template back to the previous exact template."""
|
|
85
|
+
do_change(apps, schema_editor, changes=TRANSFORM_MAP_CHANGES, forward=False)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def add_vss_to_child_filter(
|
|
89
|
+
apps: "apps_type", schema_editor: "BaseDatabaseSchemaEditor"
|
|
90
|
+
):
|
|
91
|
+
"""Add VSS chassis endpoint to Default Child Devices Filter idempotently."""
|
|
92
|
+
IPFabricFilter = apps.get_model("ipfabric_netbox", "IPFabricFilter")
|
|
93
|
+
IPFabricEndpoint = apps.get_model("ipfabric_netbox", "IPFabricEndpoint")
|
|
94
|
+
|
|
95
|
+
# Get the filter
|
|
96
|
+
try:
|
|
97
|
+
child_filter = IPFabricFilter.objects.using(schema_editor.connection.alias).get(
|
|
98
|
+
name="Default Child Devices Filter"
|
|
99
|
+
)
|
|
100
|
+
except IPFabricFilter.DoesNotExist:
|
|
101
|
+
# Filter doesn't exist, nothing to update
|
|
102
|
+
return
|
|
103
|
+
|
|
104
|
+
# Get the VSS chassis endpoint
|
|
105
|
+
try:
|
|
106
|
+
vss_endpoint = IPFabricEndpoint.objects.using(
|
|
107
|
+
schema_editor.connection.alias
|
|
108
|
+
).get(endpoint="/technology/platforms/vss/chassis")
|
|
109
|
+
except IPFabricEndpoint.DoesNotExist:
|
|
110
|
+
# Endpoint doesn't exist yet, skip (will be created by add_vss_chassis_endpoint)
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
# Add the endpoint to the filter if not already present
|
|
114
|
+
if not child_filter.endpoints.filter(pk=vss_endpoint.pk).exists():
|
|
115
|
+
child_filter.endpoints.add(vss_endpoint)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def remove_vss_from_child_filter(
|
|
119
|
+
apps: "apps_type", schema_editor: "BaseDatabaseSchemaEditor"
|
|
120
|
+
):
|
|
121
|
+
"""Remove VSS chassis endpoint from Default Child Devices Filter."""
|
|
122
|
+
IPFabricFilter = apps.get_model("ipfabric_netbox", "IPFabricFilter")
|
|
123
|
+
IPFabricEndpoint = apps.get_model("ipfabric_netbox", "IPFabricEndpoint")
|
|
124
|
+
|
|
125
|
+
# Get the filter
|
|
126
|
+
try:
|
|
127
|
+
child_filter = IPFabricFilter.objects.using(schema_editor.connection.alias).get(
|
|
128
|
+
name="Default Child Devices Filter"
|
|
129
|
+
)
|
|
130
|
+
except IPFabricFilter.DoesNotExist:
|
|
131
|
+
# Filter doesn't exist, nothing to update
|
|
132
|
+
return
|
|
133
|
+
|
|
134
|
+
# Get the VSS chassis endpoint
|
|
135
|
+
try:
|
|
136
|
+
vss_endpoint = IPFabricEndpoint.objects.using(
|
|
137
|
+
schema_editor.connection.alias
|
|
138
|
+
).get(endpoint="/technology/platforms/vss/chassis")
|
|
139
|
+
except IPFabricEndpoint.DoesNotExist:
|
|
140
|
+
# Endpoint doesn't exist, nothing to remove
|
|
141
|
+
return
|
|
142
|
+
|
|
143
|
+
# Remove the endpoint from the filter if present
|
|
144
|
+
if child_filter.endpoints.filter(pk=vss_endpoint.pk).exists():
|
|
145
|
+
child_filter.endpoints.remove(vss_endpoint)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class Migration(migrations.Migration):
|
|
149
|
+
dependencies = [
|
|
150
|
+
("ipfabric_netbox", "0024_finish_filters"),
|
|
151
|
+
]
|
|
152
|
+
|
|
153
|
+
operations = [
|
|
154
|
+
migrations.RunPython(
|
|
155
|
+
add_vss_chassis_endpoint,
|
|
156
|
+
remove_vss_chassis_endpoint,
|
|
157
|
+
),
|
|
158
|
+
migrations.RunPython(
|
|
159
|
+
forward_transform_maps_change,
|
|
160
|
+
revert_transform_maps_change,
|
|
161
|
+
),
|
|
162
|
+
migrations.RunPython(
|
|
163
|
+
add_vss_to_child_filter,
|
|
164
|
+
remove_vss_from_child_filter,
|
|
165
|
+
),
|
|
166
|
+
]
|
ipfabric_netbox/models.py
CHANGED
|
@@ -300,17 +300,19 @@ class IPFabricTransformMap(NetBoxModel):
|
|
|
300
300
|
qs = IPFabricTransformMap.objects.filter(
|
|
301
301
|
group=self.group,
|
|
302
302
|
target_model_id=self.target_model_id,
|
|
303
|
+
source_endpoint_id=self.source_endpoint_id,
|
|
303
304
|
)
|
|
304
305
|
if self.pk:
|
|
305
306
|
qs = qs.exclude(pk=self.pk)
|
|
306
307
|
if qs.exists():
|
|
307
308
|
err_msg = _(
|
|
308
|
-
f"A transform map with group '{self.group}'
|
|
309
|
+
f"A transform map with group '{self.group}', target model '{self.target_model}', and source endpoint '{self.source_endpoint}' already exists."
|
|
309
310
|
)
|
|
310
311
|
raise ValidationError(
|
|
311
312
|
{
|
|
312
313
|
"group": err_msg,
|
|
313
314
|
"target_model": err_msg,
|
|
315
|
+
"source_endpoint": err_msg,
|
|
314
316
|
}
|
|
315
317
|
)
|
|
316
318
|
|
|
@@ -412,7 +414,9 @@ class IPFabricTransformMap(NetBoxModel):
|
|
|
412
414
|
keys.update(
|
|
413
415
|
re.findall(r"object\.([a-zA-Z_0-9]+)(?=.*)", field.template)
|
|
414
416
|
)
|
|
415
|
-
|
|
417
|
+
# FIXME: Make it raise KeyError when key is missing during IN-68
|
|
418
|
+
# This is temporary hack to allow missing keys when syncing VSS
|
|
419
|
+
return {k: source_data.get(k) for k in keys}
|
|
416
420
|
|
|
417
421
|
def get_context(self, source_data):
|
|
418
422
|
new_data = deepcopy(source_data)
|
|
@@ -887,76 +891,91 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
|
|
|
887
891
|
"""
|
|
888
892
|
Returns a queryset of IPFabricTransformMap objects that would be used by this sync,
|
|
889
893
|
following group and default precedence logic.
|
|
894
|
+
|
|
895
|
+
Transform maps are unique by (target_model, source_endpoint) combination.
|
|
896
|
+
Groups have precedence: later groups override earlier groups and defaults.
|
|
890
897
|
"""
|
|
891
898
|
default_maps = IPFabricTransformMap.objects.filter(group__isnull=True)
|
|
892
899
|
group_ids = group_ids or []
|
|
893
|
-
|
|
900
|
+
|
|
901
|
+
# Use composite key: (target_model_id, source_endpoint_id)
|
|
902
|
+
maps_by_composite_key = {
|
|
903
|
+
(tm.target_model_id, tm.source_endpoint_id): tm for tm in default_maps
|
|
904
|
+
}
|
|
905
|
+
|
|
894
906
|
# Replace default maps with the ones from the groups, in given order.
|
|
895
907
|
if group_ids:
|
|
896
908
|
for group_id in group_ids:
|
|
897
909
|
group_maps = IPFabricTransformMap.objects.filter(group_id=group_id)
|
|
898
910
|
for tm in group_maps:
|
|
899
|
-
|
|
911
|
+
maps_by_composite_key[
|
|
912
|
+
(tm.target_model_id, tm.source_endpoint_id)
|
|
913
|
+
] = tm
|
|
914
|
+
|
|
900
915
|
return IPFabricTransformMap.objects.filter(
|
|
901
|
-
pk__in=[tm.pk for tm in
|
|
916
|
+
pk__in=[tm.pk for tm in maps_by_composite_key.values()]
|
|
902
917
|
)
|
|
903
918
|
|
|
904
919
|
@classmethod
|
|
905
|
-
def get_model_hierarchy(cls, group_ids=None) -> list[
|
|
920
|
+
def get_model_hierarchy(cls, group_ids=None) -> list["IPFabricTransformMap"]:
|
|
906
921
|
"""
|
|
907
|
-
Get
|
|
922
|
+
Get transform maps in hierarchical order based on parent relationships.
|
|
908
923
|
Uses topological sort (Kahn's algorithm) to support multiple parents.
|
|
909
|
-
|
|
924
|
+
Transform maps without parents come first, then their children, etc.
|
|
910
925
|
|
|
911
|
-
Example: IP Address has parents [Interface, VRF], so it will only be
|
|
912
|
-
processed after both Interface AND VRF have been processed.
|
|
926
|
+
Example: IP Address transform map has parents [Interface, VRF], so it will only be
|
|
927
|
+
processed after both Interface AND VRF transform maps have been processed.
|
|
928
|
+
|
|
929
|
+
Returns list of transform maps ordered by dependencies.
|
|
913
930
|
"""
|
|
914
931
|
maps = cls.get_transform_maps(group_ids)
|
|
915
932
|
|
|
916
|
-
# Build adjacency list and in-degree count
|
|
917
|
-
graph = {} #
|
|
918
|
-
in_degree = {} #
|
|
919
|
-
|
|
933
|
+
# Build adjacency list and in-degree count using transform map IDs
|
|
934
|
+
graph = {} # parent_tm_id -> [child_tm_id, ...]
|
|
935
|
+
in_degree = {} # tm_id -> count of unprocessed parents
|
|
936
|
+
tm_by_id = {} # tm_id -> transform_map
|
|
920
937
|
|
|
921
938
|
for transform_map in maps:
|
|
922
|
-
|
|
923
|
-
|
|
939
|
+
tm_id = transform_map.id
|
|
940
|
+
tm_by_id[tm_id] = transform_map
|
|
924
941
|
|
|
925
942
|
# Get all parents for this transform map
|
|
926
943
|
parent_maps = transform_map.parents.all()
|
|
927
944
|
|
|
928
945
|
# Set in-degree (number of parents)
|
|
929
|
-
in_degree[
|
|
946
|
+
in_degree[tm_id] = parent_maps.count()
|
|
930
947
|
|
|
931
948
|
# Build adjacency list (parent -> children)
|
|
932
949
|
for parent_map in parent_maps:
|
|
933
|
-
|
|
934
|
-
graph.setdefault(
|
|
950
|
+
parent_id = parent_map.id
|
|
951
|
+
graph.setdefault(parent_id, []).append(tm_id)
|
|
935
952
|
|
|
936
953
|
# Topological sort using Kahn's algorithm (BFS-based)
|
|
937
|
-
queue = [
|
|
954
|
+
queue = [tm_id for tm_id, degree in in_degree.items() if degree == 0]
|
|
938
955
|
ordered = []
|
|
939
956
|
|
|
940
957
|
while queue:
|
|
941
958
|
# Pop from front to maintain BFS/level-order
|
|
942
|
-
|
|
943
|
-
ordered.append(
|
|
959
|
+
current_tm_id = queue.pop(0)
|
|
960
|
+
ordered.append(current_tm_id)
|
|
944
961
|
|
|
945
962
|
# Reduce in-degree for all children
|
|
946
|
-
for
|
|
947
|
-
in_degree[
|
|
948
|
-
if in_degree[
|
|
949
|
-
queue.append(
|
|
963
|
+
for child_tm_id in graph.get(current_tm_id, []):
|
|
964
|
+
in_degree[child_tm_id] -= 1
|
|
965
|
+
if in_degree[child_tm_id] == 0:
|
|
966
|
+
queue.append(child_tm_id)
|
|
950
967
|
|
|
951
968
|
# Check for circular dependencies
|
|
952
969
|
if len(ordered) != len(in_degree):
|
|
953
|
-
|
|
970
|
+
unprocessed_ids = set(in_degree.keys()) - set(ordered)
|
|
971
|
+
unprocessed_maps = [tm_by_id[tm_id] for tm_id in unprocessed_ids]
|
|
954
972
|
raise ValidationError(
|
|
955
973
|
f"Circular dependency detected in transform map hierarchy. "
|
|
956
|
-
f"Unprocessed
|
|
974
|
+
f"Unprocessed maps: {', '.join(str(tm) for tm in unprocessed_maps)}"
|
|
957
975
|
)
|
|
958
976
|
|
|
959
|
-
|
|
977
|
+
# Return ordered list of transform maps
|
|
978
|
+
return [tm_by_id[tm_id] for tm_id in ordered]
|
|
960
979
|
|
|
961
980
|
def delete_scheduled_jobs(self) -> None:
|
|
962
981
|
Job.objects.filter(
|
|
@@ -1239,6 +1258,30 @@ class IPFabricIngestion(JobsMixin, models.Model):
|
|
|
1239
1258
|
statistics[model] = stats["current"] / stats["total"] * 100
|
|
1240
1259
|
else:
|
|
1241
1260
|
statistics[model] = stats["current"] / 1 * 100
|
|
1261
|
+
|
|
1262
|
+
# Sort statistics according to transform map hierarchy
|
|
1263
|
+
# This ensures consistent ordering in the progress display matching sync order
|
|
1264
|
+
try:
|
|
1265
|
+
group_ids = self.sync.parameters.get("groups", [])
|
|
1266
|
+
transform_maps = self.sync.get_model_hierarchy(group_ids=group_ids)
|
|
1267
|
+
|
|
1268
|
+
# Create ordered dict following the hierarchy
|
|
1269
|
+
ordered_statistics = {}
|
|
1270
|
+
for transform_map in transform_maps:
|
|
1271
|
+
model_string = f"{transform_map.target_model.app_label}.{transform_map.target_model.model}"
|
|
1272
|
+
if model_string in statistics:
|
|
1273
|
+
ordered_statistics[model_string] = statistics[model_string]
|
|
1274
|
+
|
|
1275
|
+
# Add any remaining statistics that weren't in the hierarchy
|
|
1276
|
+
for model_string, value in statistics.items():
|
|
1277
|
+
if model_string not in ordered_statistics:
|
|
1278
|
+
ordered_statistics[model_string] = value
|
|
1279
|
+
|
|
1280
|
+
statistics = ordered_statistics
|
|
1281
|
+
except Exception:
|
|
1282
|
+
# If hierarchy ordering fails, fall back to alphabetical sorting
|
|
1283
|
+
statistics = dict(sorted(statistics.items()))
|
|
1284
|
+
|
|
1242
1285
|
return {"job_results": job_results, "statistics": statistics}
|
|
1243
1286
|
|
|
1244
1287
|
def sync_merge(self):
|
|
@@ -45,10 +45,13 @@ def sort_parameters_hierarchical(
|
|
|
45
45
|
try:
|
|
46
46
|
group_ids = parameters_dict.get("groups", [])
|
|
47
47
|
|
|
48
|
-
# Get the hierarchical model order from IPFabricSync
|
|
49
|
-
|
|
50
|
-
# Convert
|
|
51
|
-
hierarchy_order = [
|
|
48
|
+
# Get the hierarchical model order from IPFabricSync (returns transform maps)
|
|
49
|
+
transform_maps = sync_obj.__class__.get_model_hierarchy(group_ids)
|
|
50
|
+
# Convert transform maps to app_label.model format
|
|
51
|
+
hierarchy_order = [
|
|
52
|
+
f"{tm.target_model.app_label}.{tm.target_model.model}"
|
|
53
|
+
for tm in transform_maps
|
|
54
|
+
]
|
|
52
55
|
|
|
53
56
|
# Group models by app label while maintaining hierarchical order
|
|
54
57
|
app_models = OrderedDict()
|