ipfabric_netbox 4.3.2b9__py3-none-any.whl → 4.3.2b11__py3-none-any.whl

ipfabric_netbox/models.py

@@ -24,7 +24,10 @@ from django.core.validators import MinValueValidator
 from django.db import models
 from django.db import transaction
 from django.db.models import Q
+from django.db.models import QuerySet
 from django.db.models import signals
+from django.db.models.signals import m2m_changed
+from django.dispatch import receiver
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.module_loading import import_string
@@ -42,19 +45,20 @@ from netbox_branching.utilities import supports_branching
 from utilities.querysets import RestrictedQuerySet
 from utilities.request import NetBoxFakeRequest
 
+from .choices import IPFabricEndpointChoices
+from .choices import IPFabricFilterTypeChoices
 from .choices import IPFabricRawDataTypeChoices
 from .choices import IPFabricSnapshotStatusModelChoices
 from .choices import IPFabricSourceStatusChoices
 from .choices import IPFabricSourceTypeChoices
 from .choices import IPFabricSyncStatusChoices
-from .choices import IPFabricTransformMapSourceModelChoices
 from .choices import required_transform_map_contenttypes
 from .signals import assign_primary_mac_address
 from .utilities.ipfutils import IPFabric
 from .utilities.ipfutils import IPFabricSyncRunner
 from .utilities.ipfutils import render_jinja2
 from .utilities.logging import SyncLogging
-
+from .utilities.transform_map import has_cycle_dfs
 
 logger = logging.getLogger("ipfabric_netbox.models")
 
@@ -98,6 +102,125 @@ IPFabricRelationshipFieldSourceModels = Q(
 )
 
 
+class IPFabricEndpoint(NetBoxModel):
+    objects = RestrictedQuerySet.as_manager()
+
+    name = models.CharField(max_length=100, unique=True)
+    description = models.TextField(blank=True, null=True)
+    endpoint = models.CharField(
+        max_length=200,
+        verbose_name=_(
+            "Endpoint path from URL notation, for example `/inventory/devices`."
+        ),
+        choices=IPFabricEndpointChoices,
+        unique=True,
+    )
+
+    class Meta:
+        ordering = ("pk",)
+        verbose_name = _("IP Fabric Endpoint")
+        verbose_name_plural = _("IP Fabric Endpoints")
+
+    def __str__(self):
+        return f"{self.endpoint}"
+
+    def get_absolute_url(self):
+        return reverse("plugins:ipfabric_netbox:ipfabricendpoint", args=[self.pk])
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if not self.endpoint.startswith("/"):
+            self.endpoint = f"/{self.endpoint}"
+        if self.endpoint.endswith("/"):
+            self.endpoint = self.endpoint.rstrip("/")
+
+    @staticmethod
+    def _merge_filter_structures(base: dict, new: dict) -> dict:
+        """Recursively merge filter structures with matching and/or keys at same level.
+
+        Args:
+            base: Base filter dictionary to merge into
+            new: New filter dictionary to merge from
+
+        Returns:
+            Merged filter dictionary
+        """
+        for key, value in new.items():
+            # Only merge 'and' and 'or' keys
+            if key not in ("and", "or") or not isinstance(value, list):
+                continue
+
+            if key not in base:
+                base[key] = []
+
+            # Process each item in the new filter's array
+            for new_item in value:
+                if not isinstance(new_item, dict):
+                    # Non-dict items just get appended
+                    base[key].append(new_item)
+                    continue
+
+                # Check if there's a matching structure in base to merge with
+                merged = False
+                for base_item in base[key]:
+                    if not isinstance(base_item, dict):
+                        continue
+
+                    # Check if both dicts have the same and/or keys
+                    new_keys = set(k for k in new_item.keys() if k in ("and", "or"))
+                    base_keys = set(k for k in base_item.keys() if k in ("and", "or"))
+
+                    if new_keys == base_keys and new_keys:
+                        # Matching structure found - recursively merge
+                        IPFabricEndpoint._merge_filter_structures(base_item, new_item)
+                        merged = True
+                        break
+
+                if not merged:
+                    # No matching structure found - append as new item
+                    base[key].append(new_item)
+
+        return base
+
+    def combine_filters(self, sync=None) -> dict:
+        """Combine all filters for this endpoint into a single filter dictionary.
+
+        Args:
+            sync: Optional IPFabricSync to filter by. If provided, only filters
+                  associated with that sync are included.
+
+        Returns:
+            Dict with filter types as keys (e.g., 'and', 'or') and lists of
+            expressions as values.
+        """
+        combined_filter = {}
+
+        # Get filters for this endpoint, optionally filtered by sync
+        if sync:
+            endpoint_filters = self.filters.filter(syncs=sync)
+        else:
+            endpoint_filters = self.filters.all()
+
+        for endpoint_filter in endpoint_filters:
+            filter_expressions = endpoint_filter.merge_expressions()
+
+            # Create a temporary dict with the filter type as key
+            new_filter = {endpoint_filter.filter_type: filter_expressions}
+
+            # Recursively merge the new filter into combined_filter
+            combined_filter = self._merge_filter_structures(combined_filter, new_filter)
+
+        # Sites filter is stored in sync parameters for user convenience
+        if sync and (sites := (sync.parameters or {}).get("sites")):
+            if "and" not in combined_filter:
+                combined_filter["and"] = []
+            combined_filter["and"].extend(
+                [{"or": [{"siteName": ["eq", site]} for site in sites]}]
+            )
+
+        return combined_filter
+
+
 class IPFabricTransformMapGroup(NetBoxModel):
     name = models.CharField(max_length=100, unique=True)
     description = models.TextField(blank=True, null=True)
@@ -117,9 +240,12 @@ class IPFabricTransformMapGroup(NetBoxModel):
 
 
 class IPFabricTransformMap(NetBoxModel):
-    name = models.CharField(max_length=100)
-    source_model = models.CharField(
-        max_length=50, choices=IPFabricTransformMapSourceModelChoices
+    name = models.CharField(max_length=200)
+    source_endpoint = models.ForeignKey(
+        to=IPFabricEndpoint,
+        on_delete=models.PROTECT,
+        related_name="transform_maps",
+        editable=True,
     )
     target_model = models.ForeignKey(
         to=ContentType,
@@ -138,6 +264,15 @@ class IPFabricTransformMap(NetBoxModel):
         blank=True,
         null=True,
     )
+    parents = models.ManyToManyField(
+        "self",
+        symmetrical=False,
+        blank=True,
+        related_name="children",
+        help_text=_(
+            "Parent transform maps, for hierarchical organization during sync."
+        ),
+    )
 
     class Meta:
         ordering = ("pk",)
@@ -145,10 +280,12 @@ class IPFabricTransformMap(NetBoxModel):
         verbose_name_plural = _("IP Fabric Transform Maps")
 
     def __str__(self):
-        if self.source_model and self.target_model:
-            return f"{self.source_model} - {self.target_model}"
-        else:
-            return "Transform Map"
+        try:
+            if self.source_endpoint and self.target_model:
+                return f"{self.source_endpoint} - {self.target_model}"
+        except (AttributeError, IPFabricEndpoint.DoesNotExist):
+            pass
+        return f"Transform Map: {self.name}" if self.name else "Transform Map"
 
     def get_absolute_url(self):
         return reverse("plugins:ipfabric_netbox:ipfabrictransformmap", args=[self.pk])
@@ -163,23 +300,51 @@ class IPFabricTransformMap(NetBoxModel):
         qs = IPFabricTransformMap.objects.filter(
             group=self.group,
             target_model_id=self.target_model_id,
+            source_endpoint_id=self.source_endpoint_id,
         )
         if self.pk:
             qs = qs.exclude(pk=self.pk)
         if qs.exists():
             err_msg = _(
-                f"A transform map with group '{self.group}' and target model '{self.target_model}' already exists."
+                f"A transform map with group '{self.group}', target model '{self.target_model}', and source endpoint '{self.source_endpoint}' already exists."
             )
             raise ValidationError(
                 {
                     "group": err_msg,
                     "target_model": err_msg,
+                    "source_endpoint": err_msg,
                 }
             )
+
+        # Validate no circular dependencies (only if saved and has parents)
+        if self.pk:
+            self._validate_no_circular_dependency()
+
         return cleaned_data
 
+    def _validate_no_circular_dependency(self):
+        """
+        Check if the current parent relationships create a circular dependency.
+        Uses DFS to detect cycles in the directed graph.
+        """
+
+        def get_parents(node_id: int, parent_override: list | None) -> models.QuerySet:
+            """Get parents for a node."""
+            node = IPFabricTransformMap.objects.get(pk=node_id)
+            return node.parents.all()
+
+        if has_cycle_dfs(self.pk, get_parents):
+            raise ValidationError(
+                {
+                    "parents": _(
+                        "The selected parents create a circular dependency. "
+                        "A transform map cannot be an ancestor of itself."
+                    )
+                }
+            )
+
     @functools.cache
-    def get_models(self):
+    def get_all_models(self):
         _context = dict()
 
         for app, app_models in apps.all_models.items():
@@ -194,6 +359,13 @@ class IPFabricTransformMap(NetBoxModel):
         _context["contenttypes"]["ContentType"] = ContentType
         return _context
 
+    @classmethod
+    def get_distinct_target_models(cls) -> QuerySet[ContentType]:
+        target_model_ids = IPFabricTransformMap.objects.values_list(
+            "target_model", flat=True
+        ).distinct()
+        return ContentType.objects.filter(id__in=target_model_ids)
+
     def build_relationships(self, source_data):
         relationship_maps = self.relationship_maps.all()
         rel_dict = {}
@@ -205,7 +377,7 @@ class IPFabricTransformMap(NetBoxModel):
             context = {
                 "object": source_data,
             }
-            context.update(self.get_models())
+            context.update(self.get_all_models())
             text = render_jinja2(field.template, context).strip()
             if text:
                 try:
@@ -242,7 +414,9 @@ class IPFabricTransformMap(NetBoxModel):
                 keys.update(
                     re.findall(r"object\.([a-zA-Z_0-9]+)(?=.*)", field.template)
                 )
-        return {k: source_data[k] for k in keys}
+        # FIXME: Make it raise KeyError when key is missing during IN-68
+        #  This is temporary hack to allow missing keys when syncing VSS
+        return {k: source_data.get(k) for k in keys}
 
     def get_context(self, source_data):
         new_data = deepcopy(source_data)
@@ -306,7 +480,7 @@ class IPFabricTransformMap(NetBoxModel):
                     "object": source_data,
                     field.source_field: source_data[field.source_field],
                 }
-                context.update(self.get_models())
+                context.update(self.get_all_models())
                 text = render_jinja2(field.template, context).strip()
             else:
                 text = source_data[field.source_field]
@@ -717,20 +891,92 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
         """
         Returns a queryset of IPFabricTransformMap objects that would be used by this sync,
         following group and default precedence logic.
+
+        Transform maps are unique by (target_model, source_endpoint) combination.
+        Groups have precedence: later groups override earlier groups and defaults.
         """
         default_maps = IPFabricTransformMap.objects.filter(group__isnull=True)
         group_ids = group_ids or []
-        maps_by_target = {tm.target_model_id: tm for tm in default_maps}
+
+        # Use composite key: (target_model_id, source_endpoint_id)
+        maps_by_composite_key = {
+            (tm.target_model_id, tm.source_endpoint_id): tm for tm in default_maps
+        }
+
         # Replace default maps with the ones from the groups, in given order.
         if group_ids:
             for group_id in group_ids:
                 group_maps = IPFabricTransformMap.objects.filter(group_id=group_id)
                 for tm in group_maps:
-                    maps_by_target[tm.target_model_id] = tm
+                    maps_by_composite_key[
+                        (tm.target_model_id, tm.source_endpoint_id)
+                    ] = tm
+
         return IPFabricTransformMap.objects.filter(
-            pk__in=[tm.pk for tm in maps_by_target.values()]
+            pk__in=[tm.pk for tm in maps_by_composite_key.values()]
         )
 
+    @classmethod
+    def get_model_hierarchy(cls, group_ids=None) -> list["IPFabricTransformMap"]:
+        """
+        Get transform maps in hierarchical order based on parent relationships.
+        Uses topological sort (Kahn's algorithm) to support multiple parents.
+        Transform maps without parents come first, then their children, etc.
+
+        Example: IP Address transform map has parents [Interface, VRF], so it will only be
+        processed after both Interface AND VRF transform maps have been processed.
+
+        Returns list of transform maps ordered by dependencies.
+        """
+        maps = cls.get_transform_maps(group_ids)
+
+        # Build adjacency list and in-degree count using transform map IDs
+        graph = {}  # parent_tm_id -> [child_tm_id, ...]
+        in_degree = {}  # tm_id -> count of unprocessed parents
+        tm_by_id = {}  # tm_id -> transform_map
+
+        for transform_map in maps:
+            tm_id = transform_map.id
+            tm_by_id[tm_id] = transform_map
+
+            # Get all parents for this transform map
+            parent_maps = transform_map.parents.all()
+
+            # Set in-degree (number of parents)
+            in_degree[tm_id] = parent_maps.count()
+
+            # Build adjacency list (parent -> children)
+            for parent_map in parent_maps:
+                parent_id = parent_map.id
+                graph.setdefault(parent_id, []).append(tm_id)
+
+        # Topological sort using Kahn's algorithm (BFS-based)
+        queue = [tm_id for tm_id, degree in in_degree.items() if degree == 0]
+        ordered = []
+
+        while queue:
+            # Pop from front to maintain BFS/level-order
+            current_tm_id = queue.pop(0)
+            ordered.append(current_tm_id)
+
+            # Reduce in-degree for all children
+            for child_tm_id in graph.get(current_tm_id, []):
+                in_degree[child_tm_id] -= 1
+                if in_degree[child_tm_id] == 0:
+                    queue.append(child_tm_id)
+
+        # Check for circular dependencies
+        if len(ordered) != len(in_degree):
+            unprocessed_ids = set(in_degree.keys()) - set(ordered)
+            unprocessed_maps = [tm_by_id[tm_id] for tm_id in unprocessed_ids]
+            raise ValidationError(
+                f"Circular dependency detected in transform map hierarchy. "
+                f"Unprocessed maps: {', '.join(str(tm) for tm in unprocessed_maps)}"
+            )
+
+        # Return ordered list of transform maps
+        return [tm_by_id[tm_id] for tm_id in ordered]
+
     def delete_scheduled_jobs(self) -> None:
         Job.objects.filter(
             object_type=ObjectType.objects.get_for_model(self),
@@ -1012,6 +1258,30 @@ class IPFabricIngestion(JobsMixin, models.Model):
                     statistics[model] = stats["current"] / stats["total"] * 100
                 else:
                     statistics[model] = stats["current"] / 1 * 100
+
+        # Sort statistics according to transform map hierarchy
+        # This ensures consistent ordering in the progress display matching sync order
+        try:
+            group_ids = self.sync.parameters.get("groups", [])
+            transform_maps = self.sync.get_model_hierarchy(group_ids=group_ids)
+
+            # Create ordered dict following the hierarchy
+            ordered_statistics = {}
+            for transform_map in transform_maps:
+                model_string = f"{transform_map.target_model.app_label}.{transform_map.target_model.model}"
+                if model_string in statistics:
+                    ordered_statistics[model_string] = statistics[model_string]
+
+            # Add any remaining statistics that weren't in the hierarchy
+            for model_string, value in statistics.items():
+                if model_string not in ordered_statistics:
+                    ordered_statistics[model_string] = value
+
+            statistics = ordered_statistics
+        except Exception:
+            # If hierarchy ordering fails, fall back to alphabetical sorting
+            statistics = dict(sorted(statistics.items()))
+
         return {"job_results": job_results, "statistics": statistics}
 
     def sync_merge(self):
@@ -1081,3 +1351,148 @@ class IPFabricData(models.Model):
 
     def get_absolute_url(self):
         return reverse("plugins:ipfabric_netbox:ipfabricdata_data", args=[self.pk])
+
+
+class IPFabricFilter(NetBoxModel):
+    objects = RestrictedQuerySet.as_manager()
+
+    name = models.CharField(max_length=100, unique=True)
+    description = models.TextField(blank=True, null=True)
+    endpoints = models.ManyToManyField(
+        to=IPFabricEndpoint,
+        related_name="filters",
+        editable=True,
+        default=None,
+        blank=True,
+    )
+    filter_type = models.CharField(
+        max_length=10, choices=IPFabricFilterTypeChoices, verbose_name=_("Filter Type")
+    )
+    syncs = models.ManyToManyField(
+        to=IPFabricSync,
+        related_name="filters",
+        editable=True,
+        default=None,
+        blank=True,
+    )
+
+    class Meta:
+        ordering = ("pk",)
+        verbose_name = _("IP Fabric Filter")
+        verbose_name_plural = _("IP Fabric Filters")
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse("plugins:ipfabric_netbox:ipfabricfilter", args=[self.pk])
+
+    def merge_expressions(self) -> list[dict]:
+        """Merge all linked Expressions into a single filter expression."""
+        merged_expression = []
+        for expression in self.expressions.all():
+            merged_expression.extend(expression.expression)
+        return merged_expression
+
+
+class IPFabricFilterExpression(NetBoxModel):
+    objects = RestrictedQuerySet.as_manager()
+
+    name = models.CharField(max_length=100, unique=True)
+    description = models.TextField(blank=True, null=True)
+    expression = models.JSONField(
+        blank=False,
+        null=False,
+        default=list,
+        verbose_name=_("IP Fabric Filter Expression JSON"),
+        help_text=_(
+            "JSON filter for API call to IPF, can be obtained from IPF UI call via browser developer console."
+        ),
+    )
+    filters = models.ManyToManyField(
+        to=IPFabricFilter,
+        related_name="expressions",
+        editable=True,
+    )
+
+    class Meta:
+        ordering = ("pk",)
+        verbose_name = _("IP Fabric Filter Expression")
+        verbose_name_plural = _("IP Fabric Filter Expressions")
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse(
+            "plugins:ipfabric_netbox:ipfabricfilterexpression", args=[self.pk]
+        )
+
+    def clean(self):
+        super().clean()
+
+        # Validate that expression is a list of dictionaries
+        if self.expression is None:
+            raise ValidationError({"expression": _("Filter Expression is required.")})
+
+        if not isinstance(self.expression, list):
+            raise ValidationError(
+                {
+                    "expression": _("Expression must be a list. Got: %(type)s")
+                    % {"type": type(self.expression).__name__}
+                }
+            )
+
+        if not self.expression:
+            raise ValidationError(
+                {"expression": _("Expression cannot be an empty list.")}
+            )
+
+        for idx, item in enumerate(self.expression):
+            if not isinstance(item, dict):
+                raise ValidationError(
+                    {
+                        "expression": _(
+                            "Expression item at index %(index)d must be a dictionary. Got: %(type)s"
+                        )
+                        % {"index": idx, "type": type(item).__name__}
+                    }
+                )
+
+
+@receiver(m2m_changed, sender=IPFabricTransformMap.parents.through)
+def validate_circular_dependency_on_m2m_change(
+    sender, instance, action, pk_set, **kwargs
+):
+    """
+    Validate circular dependencies when parent M2M relationships are modified.
+    This catches changes made through the API or programmatically.
+    """
+    if action == "pre_add" and pk_set:
+        # Simulate what the parents would be after this add operation
+        current_parent_ids = set(instance.parents.values_list("pk", flat=True))
+        future_parent_ids = current_parent_ids | pk_set
+
+        # Get the actual parent objects
+        future_parents = IPFabricTransformMap.objects.filter(pk__in=future_parent_ids)
+
+        # Run cycle detection with the future parent set
+        def get_parents(
+            node_id: int, parent_override: models.QuerySet | None
+        ) -> models.QuerySet:
+            """Get parents for a node, with optional override for the instance being modified."""
+            if node_id == instance.pk and parent_override is not None:
+                # Use the future parents for the current node
+                return parent_override
+            else:
+                # Use existing parents for other nodes
+                node = IPFabricTransformMap.objects.get(pk=node_id)
+                return node.parents.all()
+
+        if has_cycle_dfs(instance.pk, get_parents, parent_override=future_parents):
+            raise ValidationError(
+                _(
+                    "Cannot add these parents: circular dependency detected. "
+                    "A transform map cannot be an ancestor of itself."
+                )
+            )