ipfabric_netbox 4.3.2b9__py3-none-any.whl → 4.3.2b10__py3-none-any.whl

ipfabric_netbox/models.py

@@ -24,7 +24,10 @@ from django.core.validators import MinValueValidator
 from django.db import models
 from django.db import transaction
 from django.db.models import Q
+from django.db.models import QuerySet
 from django.db.models import signals
+from django.db.models.signals import m2m_changed
+from django.dispatch import receiver
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.module_loading import import_string
@@ -42,19 +45,20 @@ from netbox_branching.utilities import supports_branching
 from utilities.querysets import RestrictedQuerySet
 from utilities.request import NetBoxFakeRequest
 
+from .choices import IPFabricEndpointChoices
+from .choices import IPFabricFilterTypeChoices
 from .choices import IPFabricRawDataTypeChoices
 from .choices import IPFabricSnapshotStatusModelChoices
 from .choices import IPFabricSourceStatusChoices
 from .choices import IPFabricSourceTypeChoices
 from .choices import IPFabricSyncStatusChoices
-from .choices import IPFabricTransformMapSourceModelChoices
 from .choices import required_transform_map_contenttypes
 from .signals import assign_primary_mac_address
 from .utilities.ipfutils import IPFabric
 from .utilities.ipfutils import IPFabricSyncRunner
 from .utilities.ipfutils import render_jinja2
 from .utilities.logging import SyncLogging
-
+from .utilities.transform_map import has_cycle_dfs
 
 logger = logging.getLogger("ipfabric_netbox.models")
 
@@ -98,6 +102,125 @@ IPFabricRelationshipFieldSourceModels = Q(
 )
 
 
+class IPFabricEndpoint(NetBoxModel):
+    objects = RestrictedQuerySet.as_manager()
+
+    name = models.CharField(max_length=100, unique=True)
+    description = models.TextField(blank=True, null=True)
+    endpoint = models.CharField(
+        max_length=200,
+        verbose_name=_(
+            "Endpoint path from URL notation, for example `/inventory/devices`."
+        ),
+        choices=IPFabricEndpointChoices,
+        unique=True,
+    )
+
+    class Meta:
+        ordering = ("pk",)
+        verbose_name = _("IP Fabric Endpoint")
+        verbose_name_plural = _("IP Fabric Endpoints")
+
+    def __str__(self):
+        return f"{self.endpoint}"
+
+    def get_absolute_url(self):
+        return reverse("plugins:ipfabric_netbox:ipfabricendpoint", args=[self.pk])
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if not self.endpoint.startswith("/"):
+            self.endpoint = f"/{self.endpoint}"
+        if self.endpoint.endswith("/"):
+            self.endpoint = self.endpoint.rstrip("/")
+
+    @staticmethod
+    def _merge_filter_structures(base: dict, new: dict) -> dict:
+        """Recursively merge filter structures with matching and/or keys at same level.
+
+        Args:
+            base: Base filter dictionary to merge into
+            new: New filter dictionary to merge from
+
+        Returns:
+            Merged filter dictionary
+        """
+        for key, value in new.items():
+            # Only merge 'and' and 'or' keys
+            if key not in ("and", "or") or not isinstance(value, list):
+                continue
+
+            if key not in base:
+                base[key] = []
+
+            # Process each item in the new filter's array
+            for new_item in value:
+                if not isinstance(new_item, dict):
+                    # Non-dict items just get appended
+                    base[key].append(new_item)
+                    continue
+
+                # Check if there's a matching structure in base to merge with
+                merged = False
+                for base_item in base[key]:
+                    if not isinstance(base_item, dict):
+                        continue
+
+                    # Check if both dicts have the same and/or keys
+                    new_keys = set(k for k in new_item.keys() if k in ("and", "or"))
+                    base_keys = set(k for k in base_item.keys() if k in ("and", "or"))
+
+                    if new_keys == base_keys and new_keys:
+                        # Matching structure found - recursively merge
+                        IPFabricEndpoint._merge_filter_structures(base_item, new_item)
+                        merged = True
+                        break
+
+                if not merged:
+                    # No matching structure found - append as new item
+                    base[key].append(new_item)
+
+        return base
+
+    def combine_filters(self, sync=None) -> dict:
+        """Combine all filters for this endpoint into a single filter dictionary.
+
+        Args:
+            sync: Optional IPFabricSync to filter by. If provided, only filters
+                  associated with that sync are included.
+
+        Returns:
+            Dict with filter types as keys (e.g., 'and', 'or') and lists of
+            expressions as values.
+        """
+        combined_filter = {}
+
+        # Get filters for this endpoint, optionally filtered by sync
+        if sync:
+            endpoint_filters = self.filters.filter(syncs=sync)
+        else:
+            endpoint_filters = self.filters.all()
+
+        for endpoint_filter in endpoint_filters:
+            filter_expressions = endpoint_filter.merge_expressions()
+
+            # Create a temporary dict with the filter type as key
+            new_filter = {endpoint_filter.filter_type: filter_expressions}
+
+            # Recursively merge the new filter into combined_filter
+            combined_filter = self._merge_filter_structures(combined_filter, new_filter)
+
+        # Sites filter is stored in sync parameters for user convenience
+        if sync and (sites := (sync.parameters or {}).get("sites")):
+            if "and" not in combined_filter:
+                combined_filter["and"] = []
+            combined_filter["and"].extend(
+                [{"or": [{"siteName": ["eq", site]} for site in sites]}]
+            )
+
+        return combined_filter
+
+
 class IPFabricTransformMapGroup(NetBoxModel):
     name = models.CharField(max_length=100, unique=True)
     description = models.TextField(blank=True, null=True)
@@ -117,9 +240,12 @@ class IPFabricTransformMapGroup(NetBoxModel):
 
 
 class IPFabricTransformMap(NetBoxModel):
-    name = models.CharField(max_length=100)
-    source_model = models.CharField(
-        max_length=50, choices=IPFabricTransformMapSourceModelChoices
+    name = models.CharField(max_length=200)
+    source_endpoint = models.ForeignKey(
+        to=IPFabricEndpoint,
+        on_delete=models.PROTECT,
+        related_name="transform_maps",
+        editable=True,
     )
     target_model = models.ForeignKey(
         to=ContentType,
@@ -138,6 +264,15 @@ class IPFabricTransformMap(NetBoxModel):
         blank=True,
         null=True,
     )
+    parents = models.ManyToManyField(
+        "self",
+        symmetrical=False,
+        blank=True,
+        related_name="children",
+        help_text=_(
+            "Parent transform maps, for hierarchical organization during sync."
+        ),
+    )
 
     class Meta:
         ordering = ("pk",)
@@ -145,10 +280,12 @@ class IPFabricTransformMap(NetBoxModel):
         verbose_name_plural = _("IP Fabric Transform Maps")
 
     def __str__(self):
-        if self.source_model and self.target_model:
-            return f"{self.source_model} - {self.target_model}"
-        else:
-            return "Transform Map"
+        try:
+            if self.source_endpoint and self.target_model:
+                return f"{self.source_endpoint} - {self.target_model}"
+        except (AttributeError, IPFabricEndpoint.DoesNotExist):
+            pass
+        return f"Transform Map: {self.name}" if self.name else "Transform Map"
 
     def get_absolute_url(self):
         return reverse("plugins:ipfabric_netbox:ipfabrictransformmap", args=[self.pk])
@@ -176,10 +313,36 @@ class IPFabricTransformMap(NetBoxModel):
                     "target_model": err_msg,
                 }
             )
+
+        # Validate no circular dependencies (only if saved and has parents)
+        if self.pk:
+            self._validate_no_circular_dependency()
+
         return cleaned_data
 
+    def _validate_no_circular_dependency(self):
+        """
+        Check if the current parent relationships create a circular dependency.
+        Uses DFS to detect cycles in the directed graph.
+        """
+
+        def get_parents(node_id: int, parent_override: list | None) -> models.QuerySet:
+            """Get parents for a node."""
+            node = IPFabricTransformMap.objects.get(pk=node_id)
+            return node.parents.all()
+
+        if has_cycle_dfs(self.pk, get_parents):
+            raise ValidationError(
+                {
+                    "parents": _(
+                        "The selected parents create a circular dependency. "
+                        "A transform map cannot be an ancestor of itself."
+                    )
+                }
+            )
+
     @functools.cache
-    def get_models(self):
+    def get_all_models(self):
         _context = dict()
 
         for app, app_models in apps.all_models.items():
@@ -194,6 +357,13 @@ class IPFabricTransformMap(NetBoxModel):
         _context["contenttypes"]["ContentType"] = ContentType
         return _context
 
+    @classmethod
+    def get_distinct_target_models(cls) -> QuerySet[ContentType]:
+        target_model_ids = IPFabricTransformMap.objects.values_list(
+            "target_model", flat=True
+        ).distinct()
+        return ContentType.objects.filter(id__in=target_model_ids)
+
     def build_relationships(self, source_data):
         relationship_maps = self.relationship_maps.all()
         rel_dict = {}
@@ -205,7 +375,7 @@ class IPFabricTransformMap(NetBoxModel):
             context = {
                 "object": source_data,
             }
-            context.update(self.get_models())
+            context.update(self.get_all_models())
             text = render_jinja2(field.template, context).strip()
             if text:
                 try:
@@ -306,7 +476,7 @@ class IPFabricTransformMap(NetBoxModel):
                     "object": source_data,
                     field.source_field: source_data[field.source_field],
                 }
-                context.update(self.get_models())
+                context.update(self.get_all_models())
                 text = render_jinja2(field.template, context).strip()
             else:
                 text = source_data[field.source_field]
@@ -731,6 +901,63 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
             pk__in=[tm.pk for tm in maps_by_target.values()]
         )
 
+    @classmethod
+    def get_model_hierarchy(cls, group_ids=None) -> list[ContentType]:
+        """
+        Get target models from transform maps in hierarchical order.
+        Uses topological sort (Kahn's algorithm) to support multiple parents.
+        Models without parents come first, then their children, etc.
+
+        Example: IP Address has parents [Interface, VRF], so it will only be
+        processed after both Interface AND VRF have been processed.
+        """
+        maps = cls.get_transform_maps(group_ids)
+
+        # Build adjacency list and in-degree count
+        graph = {}  # parent_ct -> [child_ct, ...]
+        in_degree = {}  # ct -> count of unprocessed parents
+        ct_to_map = {}  # ct -> transform_map (for reference)
+
+        for transform_map in maps:
+            ct = transform_map.target_model
+            ct_to_map[ct] = transform_map
+
+            # Get all parents for this transform map
+            parent_maps = transform_map.parents.all()
+
+            # Set in-degree (number of parents)
+            in_degree[ct] = parent_maps.count()
+
+            # Build adjacency list (parent -> children)
+            for parent_map in parent_maps:
+                parent_ct = parent_map.target_model
+                graph.setdefault(parent_ct, []).append(ct)
+
+        # Topological sort using Kahn's algorithm (BFS-based)
+        queue = [ct for ct, degree in in_degree.items() if degree == 0]
+        ordered = []
+
+        while queue:
+            # Pop from front to maintain BFS/level-order
+            current_ct = queue.pop(0)
+            ordered.append(current_ct)
+
+            # Reduce in-degree for all children
+            for child_ct in graph.get(current_ct, []):
+                in_degree[child_ct] -= 1
+                if in_degree[child_ct] == 0:
+                    queue.append(child_ct)
+
+        # Check for circular dependencies
+        if len(ordered) != len(in_degree):
+            unprocessed = set(in_degree.keys()) - set(ordered)
+            raise ValidationError(
+                f"Circular dependency detected in transform map hierarchy. "
+                f"Unprocessed models: {', '.join(str(ct) for ct in unprocessed)}"
+            )
+
+        return ordered
+
     def delete_scheduled_jobs(self) -> None:
         Job.objects.filter(
             object_type=ObjectType.objects.get_for_model(self),
@@ -1081,3 +1308,148 @@ class IPFabricData(models.Model):
 
     def get_absolute_url(self):
         return reverse("plugins:ipfabric_netbox:ipfabricdata_data", args=[self.pk])
+
+
+class IPFabricFilter(NetBoxModel):
+    objects = RestrictedQuerySet.as_manager()
+
+    name = models.CharField(max_length=100, unique=True)
+    description = models.TextField(blank=True, null=True)
+    endpoints = models.ManyToManyField(
+        to=IPFabricEndpoint,
+        related_name="filters",
+        editable=True,
+        default=None,
+        blank=True,
+    )
+    filter_type = models.CharField(
+        max_length=10, choices=IPFabricFilterTypeChoices, verbose_name=_("Filter Type")
+    )
+    syncs = models.ManyToManyField(
+        to=IPFabricSync,
+        related_name="filters",
+        editable=True,
+        default=None,
+        blank=True,
+    )
+
+    class Meta:
+        ordering = ("pk",)
+        verbose_name = _("IP Fabric Filter")
+        verbose_name_plural = _("IP Fabric Filters")
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse("plugins:ipfabric_netbox:ipfabricfilter", args=[self.pk])
+
+    def merge_expressions(self) -> list[dict]:
+        """Merge all linked Expressions into a single filter expression."""
+        merged_expression = []
+        for expression in self.expressions.all():
+            merged_expression.extend(expression.expression)
+        return merged_expression
+
+
+class IPFabricFilterExpression(NetBoxModel):
+    objects = RestrictedQuerySet.as_manager()
+
+    name = models.CharField(max_length=100, unique=True)
+    description = models.TextField(blank=True, null=True)
+    expression = models.JSONField(
+        blank=False,
+        null=False,
+        default=list,
+        verbose_name=_("IP Fabric Filter Expression JSON"),
+        help_text=_(
+            "JSON filter for API call to IPF, can be obtained from IPF UI call via browser developer console."
+        ),
+    )
+    filters = models.ManyToManyField(
+        to=IPFabricFilter,
+        related_name="expressions",
+        editable=True,
+    )
+
+    class Meta:
+        ordering = ("pk",)
+        verbose_name = _("IP Fabric Filter Expression")
+        verbose_name_plural = _("IP Fabric Filter Expressions")
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse(
+            "plugins:ipfabric_netbox:ipfabricfilterexpression", args=[self.pk]
+        )
+
+    def clean(self):
+        super().clean()
+
+        # Validate that expression is a list of dictionaries
+        if self.expression is None:
+            raise ValidationError({"expression": _("Filter Expression is required.")})
+
+        if not isinstance(self.expression, list):
+            raise ValidationError(
+                {
+                    "expression": _("Expression must be a list. Got: %(type)s")
+                    % {"type": type(self.expression).__name__}
+                }
+            )
+
+        if not self.expression:
+            raise ValidationError(
+                {"expression": _("Expression cannot be an empty list.")}
+            )
+
+        for idx, item in enumerate(self.expression):
+            if not isinstance(item, dict):
+                raise ValidationError(
+                    {
+                        "expression": _(
+                            "Expression item at index %(index)d must be a dictionary. Got: %(type)s"
+                        )
+                        % {"index": idx, "type": type(item).__name__}
+                    }
+                )
+
+
+@receiver(m2m_changed, sender=IPFabricTransformMap.parents.through)
+def validate_circular_dependency_on_m2m_change(
+    sender, instance, action, pk_set, **kwargs
+):
+    """
+    Validate circular dependencies when parent M2M relationships are modified.
+    This catches changes made through the API or programmatically.
+    """
+    if action == "pre_add" and pk_set:
+        # Simulate what the parents would be after this add operation
+        current_parent_ids = set(instance.parents.values_list("pk", flat=True))
+        future_parent_ids = current_parent_ids | pk_set
+
+        # Get the actual parent objects
+        future_parents = IPFabricTransformMap.objects.filter(pk__in=future_parent_ids)
+
+        # Run cycle detection with the future parent set
+        def get_parents(
+            node_id: int, parent_override: models.QuerySet | None
+        ) -> models.QuerySet:
+            """Get parents for a node, with optional override for the instance being modified."""
+            if node_id == instance.pk and parent_override is not None:
+                # Use the future parents for the current node
+                return parent_override
+            else:
+                # Use existing parents for other nodes
+                node = IPFabricTransformMap.objects.get(pk=node_id)
+                return node.parents.all()
+
+        if has_cycle_dfs(instance.pk, get_parents, parent_override=future_parents):
+            raise ValidationError(
+                _(
+                    "Cannot add these parents: circular dependency detected. "
+                    "A transform map cannot be an ancestor of itself."
+                )
+            )

ipfabric_netbox/navigation.py

@@ -3,29 +3,17 @@ from netbox.plugins import PluginMenu
 from netbox.plugins import PluginMenuButton
 from netbox.plugins import PluginMenuItem
 
-
-sync_buttons = [
-    PluginMenuButton(
-        link="plugins:ipfabric_netbox:ipfabricsync_add",
-        title=_("Add"),
-        icon_class="mdi mdi-plus-thick",
-        permissions=["ipfabric_netbox.add_ipfabricsync"],
-    )
-]
-
-source_buttons = [
-    PluginMenuButton(
-        link="plugins:ipfabric_netbox:ipfabricsource_add",
-        title=_("Add"),
-        icon_class="mdi mdi-plus-thick",
-        permissions=["ipfabric_netbox.add_ipfabricsource"],
-    )
-]
-
 source = PluginMenuItem(
     link="plugins:ipfabric_netbox:ipfabricsource_list",
     link_text=_("Sources"),
-    buttons=source_buttons,
+    buttons=[
+        PluginMenuButton(
+            link="plugins:ipfabric_netbox:ipfabricsource_add",
+            title=_("Add"),
+            icon_class="mdi mdi-plus-thick",
+            permissions=["ipfabric_netbox.add_ipfabricsource"],
+        )
+    ],
     permissions=["ipfabric_netbox.view_ipfabricsource"],
 )
 
@@ -35,14 +23,62 @@ snapshot = PluginMenuItem(
     permissions=["ipfabric_netbox.view_ipfabricsnapshot"],
 )
 
-
-ingestion = PluginMenuItem(
+sync = PluginMenuItem(
     link="plugins:ipfabric_netbox:ipfabricsync_list",
     link_text=_("Syncs"),
-    buttons=sync_buttons,
+    buttons=[
+        PluginMenuButton(
+            link="plugins:ipfabric_netbox:ipfabricsync_add",
+            title=_("Add"),
+            icon_class="mdi mdi-plus-thick",
+            permissions=["ipfabric_netbox.add_ipfabricsync"],
+        )
+    ],
     permissions=["ipfabric_netbox.view_ipfabricsync"],
 )
 
+ingestion = PluginMenuItem(
+    link="plugins:ipfabric_netbox:ipfabricingestion_list",
+    link_text=_("Ingestions"),
+    buttons=[],
+    permissions=["ipfabric_netbox.view_ipfabricingestion"],
+)
+
+endpoint = PluginMenuItem(
+    link="plugins:ipfabric_netbox:ipfabricendpoint_list",
+    link_text=_("Endpoints"),
+    permissions=["ipfabric_netbox.view_ipfabricendpoint"],
+    buttons=[],
+)
+
+filter = PluginMenuItem(
+    link="plugins:ipfabric_netbox:ipfabricfilter_list",
+    link_text=_("Filters"),
+    permissions=["ipfabric_netbox.view_ipfabricfilter"],
+    buttons=[
+        PluginMenuButton(
+            link="plugins:ipfabric_netbox:ipfabricfilter_add",
+            title=_("Add"),
+            icon_class="mdi mdi-plus-thick",
+            permissions=["ipfabric_netbox.add_ipfabricfilter"],
+        )
+    ],
+)
+
+filter_expression = PluginMenuItem(
+    link="plugins:ipfabric_netbox:ipfabricfilterexpression_list",
+    link_text=_("Filter Expressions"),
+    permissions=["ipfabric_netbox.view_ipfabricfilterexpression"],
+    buttons=[
+        PluginMenuButton(
+            link="plugins:ipfabric_netbox:ipfabricfilterexpression_add",
+            title=_("Add"),
+            icon_class="mdi mdi-plus-thick",
+            permissions=["ipfabric_netbox.add_ipfabricfilterexpression"],
+        )
+    ],
+)
+
 tmg = PluginMenuItem(
     link="plugins:ipfabric_netbox:ipfabrictransformmapgroup_list",
     link_text=_("Transform Map Groups"),
@@ -70,8 +106,46 @@ tm = PluginMenuItem(
         )
     ],
 )
+
+tmf = PluginMenuItem(
+    link="plugins:ipfabric_netbox:ipfabrictransformfield_list",
+    link_text=_("Transform Fields"),
+    permissions=["ipfabric_netbox.view_ipfabrictransformfield"],
+    buttons=[
+        PluginMenuButton(
+            link="plugins:ipfabric_netbox:ipfabrictransformfield_add",
+            title=_("Add"),
+            icon_class="mdi mdi-plus-thick",
+            permissions=["ipfabric_netbox.add_ipfabrictransformfield"],
+        )
+    ],
+)
+
+tmr = PluginMenuItem(
+    link="plugins:ipfabric_netbox:ipfabricrelationshipfield_list",
+    link_text=_("Relationship Fields"),
+    permissions=["ipfabric_netbox.view_ipfabricrelationshipfield"],
+    buttons=[
+        PluginMenuButton(
+            link="plugins:ipfabric_netbox:ipfabricrelationshipfield_add",
+            title=_("Add"),
+            icon_class="mdi mdi-plus-thick",
+            permissions=["ipfabric_netbox.add_ipfabricrelationshipfield"],
+        )
+    ],
+)
+
 menu = PluginMenu(
     label="IP Fabric",
     icon_class="mdi mdi-cloud-sync",
-    groups=(("IP Fabric", (source, snapshot, ingestion, tmg, tm)),),
+    groups=(
+        (
+            "Data Sync",
+            (source, snapshot, sync, ingestion),
+        ),
+        (
+            "Configuration",
+            (endpoint, filter, filter_expression, tmg, tm, tmf, tmr),
+        ),
+    ),
 )