nautobot 2.2.8__py3-none-any.whl → 2.3.0__py3-none-any.whl

nautobot/extras/querysets.py

@@ -1,12 +1,10 @@
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
-from django.core.cache import cache
 from django.db.models import F, Model, OuterRef, Q, Subquery
 from django.db.models.functions import JSONObject
 
 from nautobot.core.models.query_functions import EmptyGroupByJSONBAgg
 from nautobot.core.models.querysets import RestrictedQuerySet
-from nautobot.core.utils.config import get_settings_or_config
 from nautobot.extras.models.tags import TaggedItem
 
 
@@ -85,10 +83,10 @@ class ConfigContextModelQuerySet(RestrictedQuerySet):
         """
         Attach the subquery annotation to the base queryset.
 
-        Order By clause in Subquery is not guaranteed to be respected within the aggregated JSON array, which is why
-        we include "weight" and "name" into the result so that we can sort it within Python to ensure correctness.
+        Note that the underlying function of our JSONBAgg in MySQL, `JSONARRAY_AGG`, does not support an `ordering`,
+        unlike PostgreSQL's implementation. This is why we include "weight" and "name" into the result so that we can
+        sort it within Python to ensure correctness.
 
-        TODO This method does not accurately reflect location inheritance because of the reasons stated in _get_config_context_filters()
         Do not use this method by itself, use get_config_context() method directly on ConfigContextModel instead.
         """
         from nautobot.extras.models import ConfigContext
@@ -96,7 +94,6 @@ class ConfigContextModelQuerySet(RestrictedQuerySet):
         return self.annotate(
             config_context_data=Subquery(
                 ConfigContext.objects.filter(self._get_config_context_filters())
-                .order_by("weight", "name")
                 .annotate(
                     _data=EmptyGroupByJSONBAgg(
                         JSONObject(
@@ -107,15 +104,13 @@ class ConfigContextModelQuerySet(RestrictedQuerySet):
                     )
                 )
                 .values("_data")
+                .order_by()
             )
         ).distinct()
 
     def _get_config_context_filters(self):
         """
         This method is constructing the set of Q objects for the specific object types.
-        Note that locations filters are not included in the method because the filter needs the
-        ability to query the ancestors for a particular tree node for subquery and we lost it since
-        moving from mptt to django-tree-queries https://github.com/matthiask/django-tree-queries/issues/54.
         """
         tag_query_filters = {
             "object_id": OuterRef(OuterRef("pk")),
@@ -163,81 +158,50 @@ class ConfigContextModelQuerySet(RestrictedQuerySet):
 
 
 class DynamicGroupQuerySet(RestrictedQuerySet):
-    """Queryset for `DynamicGroup` objects that provides a `get_for_object` method."""
+    """Queryset for `DynamicGroup` objects that provides `get_for_object` and `get_for_model` methods."""
+
+    def get_for_model(self, model):
+        """
+        Return all DynamicGroups assignable to the given model class.
+        """
+        concrete_model = model._meta.concrete_model
+        content_type = ContentType.objects.get_for_model(concrete_model)
+        return self.filter(content_type=content_type)
 
     def get_list_for_object(self, obj, use_cache=False):
         """
-        Return a list of `DynamicGroup` assigned to the given object. As opposed to `get_for_object`
-        which will return a queryset but that is an additional query to the DB when you may just
-        want a list.
+        Return a (cached) list of `DynamicGroup` objects assigned to the given object.
 
         Args:
             obj: The object to seek dynamic groups membership by.
-            use_cache: If True, use the cache and query the database directly.
+            use_cache: Obsolete; cache is always used. If truly necessary to get up-to-the-minute accuracy, you should
+                call `[dg.update_cached_members() for dg in DynamicGroup.objects.filter(content_type=...)]` beforehand,
+                but be aware that that's potentially quite expensive computationally.
         """
-        if not isinstance(obj, Model):
-            raise TypeError(f"{obj} is not an instance of Django Model class")
-
-        # Get dynamic groups for this content_type using the discrete content_type fields to
-        # optimize the query.
-        eligible_groups = self._get_eligible_dynamic_groups(obj, use_cache=use_cache)
-
-        # Filter down to matching groups.
-        my_groups = []
-        for dynamic_group in list(eligible_groups):
-            if dynamic_group.has_member(obj, use_cache=use_cache):
-                my_groups.append(dynamic_group)
-
-        return my_groups
+        return list(self.get_for_object(obj))
 
     def get_for_object(self, obj, use_cache=False):
         """
-        Return a queryset of `DynamicGroup` objects that are assigned to the given object.
+        Return a (cached) queryset of `DynamicGroup` objects that are assigned to the given object.
 
         Args:
             obj: The object to seek dynamic groups membership by.
-            use_cache: If True, use the cache and query the database directly.
+            use_cache: Obsolete; cache is always used. If truly necessary to get up-to-the-minute accuracy, you should
+                call `[dg.update_cached_members() for dg in DynamicGroup.objects.filter(content_type=...)]` beforehand,
+                but be aware that's potentially quite expensive computationally.
         """
-        return self.filter(pk__in=[dg.pk for dg in self.get_list_for_object(obj, use_cache=use_cache)])
+        if not isinstance(obj, Model):
+            raise TypeError(f"{obj} is not an instance of Django Model class")
+
+        return self.filter(
+            content_type__app_label=obj._meta.app_label,
+            content_type__model=obj._meta.model_name,
+            static_group_associations__associated_object_id=obj.id,
+        )
 
     def get_by_natural_key(self, slug):
         return self.get(slug=slug)
 
-    @classmethod
-    def _get_eligible_dynamic_groups_cache_key(cls, obj):
-        """
-        Return the cache key for the queryset of `DynamicGroup` objects that are eligible to potentially contain the
-        given object.
-        """
-        return f"nautobot.{obj._meta.label_lower}._get_eligible_dynamic_groups"
-
-    def _get_eligible_dynamic_groups(self, obj, use_cache=False):
-        """
-        Return a queryset of `DynamicGroup` objects that are eligible to potentially contain the given object.
-        """
-        cache_key = self.__class__._get_eligible_dynamic_groups_cache_key(obj)
-
-        def _query_eligible_dynamic_groups():
-            """
-            A callable to be used as the default value for the cache of which dynamic groups are
-            eligible for a given object.
-            """
-            # Save a DB query if we can by using the _content_type field on the model which is a cached instance of the ContentType
-            if use_cache and hasattr(type(obj), "_content_type"):
-                return self.filter(content_type_id=type(obj)._content_type.id)
-            return self.filter(
-                content_type__app_label=obj._meta.app_label, content_type__model=obj._meta.model_name
-            ).select_related("content_type")
-
-        if not use_cache:
-            eligible_dynamic_groups = _query_eligible_dynamic_groups()
-            cache.set(cache_key, eligible_dynamic_groups, get_settings_or_config("DYNAMIC_GROUPS_MEMBER_CACHE_TIMEOUT"))
-            return eligible_dynamic_groups
-
-        return cache.get_or_set(
-            cache_key, _query_eligible_dynamic_groups, get_settings_or_config("DYNAMIC_GROUPS_MEMBER_CACHE_TIMEOUT")
-        )
-
 
 class DynamicGroupMembershipQuerySet(RestrictedQuerySet):
     """Queryset for `DynamicGroupMembership` objects."""

nautobot/extras/signals.py

@@ -1,9 +1,7 @@
 import contextlib
 import contextvars
-from datetime import timedelta
 import logging
 import os
-import secrets
 import shutil
 import traceback
 
@@ -15,7 +13,7 @@ from django.core.cache import cache
 from django.core.exceptions import ValidationError
 from django.core.files.storage import get_storage_class
 from django.db import transaction
-from django.db.models.signals import m2m_changed, post_delete, post_save, pre_delete, pre_save
+from django.db.models.signals import m2m_changed, post_delete, post_migrate, post_save, pre_delete, pre_save
 from django.dispatch import receiver
 from django.utils import timezone
 from django_prometheus.models import model_deletes, model_inserts, model_updates
@@ -23,7 +21,6 @@ import redis.exceptions
 
 from nautobot.core.celery import app, import_jobs
 from nautobot.core.models import BaseModel
-from nautobot.core.utils.config import get_settings_or_config
 from nautobot.core.utils.logging import sanitize
 from nautobot.extras.choices import JobResultStatusChoices, ObjectChangeActionChoices
 from nautobot.extras.constants import CHANGELOG_MAX_CHANGE_CONTEXT_DETAIL
@@ -35,6 +32,7 @@ from nautobot.extras.models import (
     DynamicGroupMembership,
     GitRepository,
     JobResult,
+    MetadataType,
     ObjectChange,
     Relationship,
 )
@@ -66,31 +64,47 @@ def get_user_if_authenticated(user, instance):
         return None
 
 
-@receiver(post_save)
-@receiver(m2m_changed)
-@receiver(post_delete)
+@receiver(post_save, sender=ComputedField)
+@receiver(post_save, sender=CustomField)
+@receiver(post_save, sender=CustomField.content_types.through)
+@receiver(post_save, sender=MetadataType)
+@receiver(post_save, sender=MetadataType.content_types.through)
+@receiver(m2m_changed, sender=ComputedField)
+@receiver(m2m_changed, sender=CustomField)
+@receiver(m2m_changed, sender=CustomField.content_types.through)
+@receiver(m2m_changed, sender=MetadataType)
+@receiver(m2m_changed, sender=MetadataType.content_types.through)
+@receiver(post_delete, sender=ComputedField)
+@receiver(post_delete, sender=CustomField)
+@receiver(post_delete, sender=CustomField.content_types.through)
+@receiver(post_delete, sender=MetadataType)
+@receiver(post_delete, sender=MetadataType.content_types.through)
 def invalidate_models_cache(sender, **kwargs):
-    """Invalidate the related-models cache for ComputedFields, CustomFields and Relationships."""
+    """Invalidate the related-models cache for ComputedFields and CustomFields."""
     if sender is CustomField.content_types.through:
         manager = CustomField.objects
-    elif sender in (ComputedField, CustomField, Relationship):
-        manager = sender.objects
+    elif sender is MetadataType.content_types.through:
+        manager = MetadataType.objects
     else:
-        return
+        manager = sender.objects
+
+    with contextlib.suppress(redis.exceptions.ConnectionError):
+        # TODO: *maybe* target more narrowly, e.g. only clear the cache for specific related content-types?
+        cache.delete_pattern(f"{manager.get_for_model.cache_key_prefix}.*")
 
-    cached_methods = (
-        "get_for_model",
-        "get_for_model_source",
-        "get_for_model_destination",
-    )
 
-    for method_name in cached_methods:
-        if hasattr(manager, method_name):
-            method = getattr(manager, method_name)
-            if hasattr(method, "cache_key_prefix"):
-                with contextlib.suppress(redis.exceptions.ConnectionError):
-                    # TODO: *maybe* target more narrowly, e.g. only clear the cache for specific related content-types?
-                    cache.delete_pattern(f"{method.cache_key_prefix}.*")
+@receiver(post_save, sender=Relationship)
+@receiver(m2m_changed, sender=Relationship)
+@receiver(post_delete, sender=Relationship)
+def invalidate_relationship_models_cache(sender, **kwargs):
+    """Invalidate the related-models caches for Relationships."""
+    for method in (
+        Relationship.objects.get_for_model_source,
+        Relationship.objects.get_for_model_destination,
+    ):
+        with contextlib.suppress(redis.exceptions.ConnectionError):
+            # TODO: *maybe* target more narrowly, e.g. only clear the cache for specific related content-types?
+            cache.delete_pattern(f"{method.cache_key_prefix}.*")
 
 
 @receiver(post_save)
@@ -131,7 +145,11 @@ def _handle_changed_object(sender, instance, raw=False, **kwargs):
 
         # Generate a unique identifier for this change to stash in the change context
         # This is used for deferred change logging and for looking up related changes without querying the database
-        unique_object_change_id = f"{changed_object_type.pk}__{changed_object_id}__{user.pk}"
+        unique_object_change_id = None
+        if user is not None:
+            unique_object_change_id = f"{changed_object_type.pk}__{changed_object_id}__{user.pk}"
+        else:
+            unique_object_change_id = f"{changed_object_type.pk}__{changed_object_id}"
 
         # If a change already exists for this change_id, user, and object, update it instead of creating a new one.
         # If the object was deleted then recreated with the same pk (don't do this), change the action to update.
@@ -146,12 +164,13 @@ def _handle_changed_object(sender, instance, raw=False, **kwargs):
             # Skip the database check when deferring object changes
             if not change_context.defer_object_changes and related_changes.exists():
                 objectchange = instance.to_objectchange(action)
-                most_recent_change = related_changes.order_by("-time").first()
-                if most_recent_change.action == ObjectChangeActionChoices.ACTION_DELETE:
-                    most_recent_change.action = ObjectChangeActionChoices.ACTION_UPDATE
-                most_recent_change.object_data = objectchange.object_data
-                most_recent_change.object_data_v2 = objectchange.object_data_v2
-                most_recent_change.save()
+                if objectchange is not None:
+                    most_recent_change = related_changes.order_by("-time").first()
+                    if most_recent_change.action == ObjectChangeActionChoices.ACTION_DELETE:
+                        most_recent_change.action = ObjectChangeActionChoices.ACTION_UPDATE
+                    most_recent_change.object_data = objectchange.object_data
+                    most_recent_change.object_data_v2 = objectchange.object_data_v2
+                    most_recent_change.save()
 
         else:
             change_context.deferred_object_changes[unique_object_change_id] = [
@@ -159,11 +178,14 @@ def _handle_changed_object(sender, instance, raw=False, **kwargs):
             ]
             if not change_context.defer_object_changes:
                 objectchange = instance.to_objectchange(action)
-                objectchange.user = user
-                objectchange.request_id = change_context.change_id
-                objectchange.change_context = change_context.context
-                objectchange.change_context_detail = change_context.context_detail[:CHANGELOG_MAX_CHANGE_CONTEXT_DETAIL]
-                objectchange.save()
+                if objectchange is not None:
+                    objectchange.user = user
+                    objectchange.request_id = change_context.change_id
+                    objectchange.change_context = change_context.context
+                    objectchange.change_context_detail = change_context.context_detail[
+                        :CHANGELOG_MAX_CHANGE_CONTEXT_DETAIL
+                    ]
+                    objectchange.save()
 
         # restore field cache
         instance._state.fields_cache = original_cache
@@ -174,12 +196,6 @@ def _handle_changed_object(sender, instance, raw=False, **kwargs):
     elif action == ObjectChangeActionChoices.ACTION_UPDATE:
         model_updates.labels(instance._meta.model_name).inc()
 
-    # Housekeeping: 0.1% chance of clearing out expired ObjectChanges
-    changelog_retention = get_settings_or_config("CHANGELOG_RETENTION")
-    if changelog_retention and secrets.randbelow(1000) == 0:
-        cutoff = timezone.now() - timedelta(days=changelog_retention)
-        ObjectChange.objects.filter(time__lt=cutoff).delete()
-
 
 @receiver(pre_delete)
 def _handle_deleted_object(sender, instance, **kwargs):
@@ -236,13 +252,14 @@ def _handle_deleted_object(sender, instance, **kwargs):
             # Skip the database check when deferring object changes
             if not change_context.defer_object_changes and related_changes.exists():
                 objectchange = instance.to_objectchange(ObjectChangeActionChoices.ACTION_DELETE)
-                most_recent_change = related_changes.order_by("-time").first()
-                if most_recent_change.action != ObjectChangeActionChoices.ACTION_CREATE:
-                    most_recent_change.action = ObjectChangeActionChoices.ACTION_DELETE
-                    most_recent_change.object_data = objectchange.object_data
-                    most_recent_change.object_data_v2 = objectchange.object_data_v2
-                    most_recent_change.save()
-                    save_new_objectchange = False
+                if objectchange is not None:
+                    most_recent_change = related_changes.order_by("-time").first()
+                    if most_recent_change.action != ObjectChangeActionChoices.ACTION_CREATE:
+                        most_recent_change.action = ObjectChangeActionChoices.ACTION_DELETE
+                        most_recent_change.object_data = objectchange.object_data
+                        most_recent_change.object_data_v2 = objectchange.object_data_v2
+                        most_recent_change.save()
+                        save_new_objectchange = False
 
         if save_new_objectchange:
             change_context.deferred_object_changes.setdefault(unique_object_change_id, []).append(
@@ -250,16 +267,20 @@ def _handle_deleted_object(sender, instance, **kwargs):
                     "action": ObjectChangeActionChoices.ACTION_DELETE,
                     "instance": instance,
                     "user": user,
-                    "changed_object_id": instance.pk,
+                    "changed_object_id": changed_object_id,
+                    "changed_object_type": changed_object_type,
                 }
             )
             if not change_context.defer_object_changes:
                 objectchange = instance.to_objectchange(ObjectChangeActionChoices.ACTION_DELETE)
-                objectchange.user = user
-                objectchange.request_id = change_context.change_id
-                objectchange.change_context = change_context.context
-                objectchange.change_context_detail = change_context.context_detail[:CHANGELOG_MAX_CHANGE_CONTEXT_DETAIL]
-                objectchange.save()
+                if objectchange is not None:
+                    objectchange.user = user
+                    objectchange.request_id = change_context.change_id
+                    objectchange.change_context = change_context.context
+                    objectchange.change_context_detail = change_context.context_detail[
+                        :CHANGELOG_MAX_CHANGE_CONTEXT_DETAIL
+                    ]
+                    objectchange.save()
 
         # restore field cache
         instance._state.fields_cache = original_cache
@@ -268,6 +289,19 @@ def _handle_deleted_object(sender, instance, **kwargs):
     model_deletes.labels(instance._meta.model_name).inc()
 
 
+#
+# Content types
+#
+
+
+@receiver(post_migrate)
+def post_migrate_clear_content_type_caches(sender, app_config, signal, **kwargs):
+    """Clear various content-type caches after a migration."""
+    with contextlib.suppress(redis.exceptions.ConnectionError):
+        cache.delete("nautobot.extras.utils.change_logged_models_queryset")
+        cache.delete_pattern("nautobot.extras.utils.FeatureQuery.*")
+
+
 #
 # Custom fields
 #
@@ -387,60 +421,18 @@ m2m_changed.connect(dynamic_group_children_changed, sender=DynamicGroup.children
 pre_save.connect(dynamic_group_membership_created, sender=DynamicGroupMembership)
 
 
-def dynamic_group_eligible_groups_changed(sender, instance, **kwargs):
-    """
-    When a DynamicGroup is created or deleted, refresh the cache of eligible groups for the associated ContentType.
-
-    Can't change content_type_id on an existing instance, so no need to check for that.
-    """
-
-    if get_settings_or_config("DYNAMIC_GROUPS_MEMBER_CACHE_TIMEOUT") == 0:
-        # Caching is disabled, so there's nothing to do
-        return
-
-    if kwargs.get("created", None) is False:
-        # We do not care about updates
-        # "created" is not a kwarg for post_delete signals, so is unset or None
-        # "created" is a kwarg for post_save signals, but you cannot change content types of existing groups
-        #   therefor we can ignore cache updates on DynamicGroups updates
-        return
-
-    content_type = instance.content_type
-    cache_key = f"nautobot.{content_type.app_label}.{content_type.model}._get_eligible_dynamic_groups"
-    cache.set(
-        cache_key,
-        DynamicGroup.objects.filter(content_type_id=instance.content_type_id),
-        get_settings_or_config("DYNAMIC_GROUPS_MEMBER_CACHE_TIMEOUT"),
-    )
-
-
-post_save.connect(dynamic_group_eligible_groups_changed, sender=DynamicGroup)
-post_delete.connect(dynamic_group_eligible_groups_changed, sender=DynamicGroup)
-
-
 def dynamic_group_update_cached_members(sender, instance, **kwargs):
     """
-    When a DynamicGroup or DynamicGroupMembership is updated, update the cache of members.
+    When a DynamicGroup or DynamicGroupMembership is updated, update the cache of members for it and any parent groups.
     """
-
-    if get_settings_or_config("DYNAMIC_GROUPS_MEMBER_CACHE_TIMEOUT") == 0:
-        # Caching is disabled, so there's nothing to do
-        return
-
     if isinstance(instance, DynamicGroupMembership):
-        group = instance.group
+        group = instance.parent_group
     else:
         group = instance
 
-    def _update_cache_and_parents(this_instance):
-        this_instance.update_cached_members()
-
-        # Since a change a group or group of groups does not affect it's children, we only need to go up the tree
-        # A group of groups does not use the cache of it's children due to the complexity of the set operations
-        for ancestor in list(this_instance.parents.all()):
-            _update_cache_and_parents(ancestor)
-
-    _update_cache_and_parents(group)
+    group.update_cached_members()
+    for ancestor in group.get_ancestors():
+        ancestor.update_cached_members()
 
 
 post_save.connect(dynamic_group_update_cached_members, sender=DynamicGroup)
@@ -494,3 +486,19 @@ def refresh_job_models(sender, *, apps, **kwargs):
             )
             job_model.installed = False
             job_model.save()
+
+
+#
+# Metadata
+#
+
+
+def handle_mdt_removed_obj_types(instance, action, pk_set, **kwargs):  # pylint: disable=useless-return
+    """Handle the cleanup of old Metadata when a MetadataType is removed from one or more ContentTypes."""
+    if action != "post_remove":
+        return
+    # Existing content types have been removed from the MetadataType, delete their data.
+    # TODO delete Metadata records with object_type in pk_set and metadata_type == instance
+
+
+m2m_changed.connect(handle_mdt_removed_obj_types, sender=MetadataType.content_types.through)