nautobot 2.3.9__py3-none-any.whl → 2.3.10__py3-none-any.whl

nautobot/core/models/query_functions.py

@@ -1,5 +1,7 @@
 from django.db import NotSupportedError
-from django.db.models import Aggregate, Func, JSONField
+from django.db.models import Aggregate, Func, JSONField, Value
+from django.db.models.fields.json import compile_json_path
+from django.db.models.functions import Cast
 
 
 class CollateAsChar(Func):
@@ -26,6 +28,150 @@ class CollateAsChar(Func):
         return super().as_sql(compiler, connection, function, template, arg_joiner, **extra_context)
 
 
+class JSONSet(Func):
+    """
+    Set or create the value of a single key in a JSONField.
+
+    Example:
+        model.objects.all().update(_custom_field_data=JSONSet("_custom_field_data", "cf_key", "new_value"))
+
+    Limitations:
+        - Postgres and MySQL only.
+        - Does *not* support nested lookups (`key1__key2`), only a single top-level key.
+        - Unlike the referenced Django PR, supports only a single key/value rather than an arbitrary number of them.
+
+    References:
+        - https://code.djangoproject.com/ticket/32519
+        - https://github.com/django/django/pull/18489/files
+    """
+
+    function = None
+
+    def __init__(self, expression, path, value, output_field=None):
+        self.path = path
+        self.value = value
+        super().__init__(expression, output_field=output_field)
+
+    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
+        """
+        Based on https://github.com/django/django/pull/18489/files.
+
+        Transforms and inserts self.path and self.value appropriately into the expression fields.
+        """
+        c = super().resolve_expression(query, allow_joins, reuse, summarize, for_save)
+        # Resolve expressions in the JSON update values.
+        c.fields = {
+            self.path: (
+                self.value.resolve_expression(query, allow_joins, reuse, summarize, for_save)
+                if hasattr(self.value, "resolve_expression")
+                else self.value
+            )
+        }
+        return c
+
+    def as_sql(self, compiler, connection, function=None, **extra_context):
+        """
+        MySQL implementation based on https://github.com/django/django/pull/18489/files.
+
+        Creates a copy of this object with the appropriately transformed self.path and self.value for MySQL JSON_SET().
+        """
+        if connection.vendor != "mysql":
+            raise NotSupportedError(f"JSONSet is not implemented for database {connection.vendor}")
+
+        copy = self.copy()
+        new_source_expressions = copy.get_source_expressions()
+
+        path = compile_json_path([self.path])
+        value = self.value
+        if not hasattr(value, "resolve_expression"):
+            # Use Value to serialize the value to a string, then Cast to ensure it's treated as JSON.
+            value = Cast(Value(value, output_field=self.output_field), output_field=self.output_field)
+
+        new_source_expressions.extend((Value(path), value))
+        copy.set_source_expressions(new_source_expressions)
+        return super(JSONSet, copy).as_sql(compiler, connection, function="JSON_SET", **extra_context)
+
+    def as_postgresql(self, compiler, connection, function=None, **extra_context):
+        """
+        PostgreSQL implementation based on https://github.com/django/django/pull/18489/files.
+
+        Creates a copy of this object with appropriately transformed self.path and self.value for Postgres JSONB_SET().
+        """
+        copy = self.copy()
+        new_source_expressions = copy.get_source_expressions()
+
+        path = self.path
+        value = self.value
+        if not hasattr(value, "resolve_expression"):
+            # We don't need Cast() here because Value with a JSONFIeld is correctly handled as JSONB by Postgres
+            value = Value(value, output_field=self.output_field)
+        else:
+
+            class ToJSONB(Func):
+                function = "TO_JSONB"
+
+            value = ToJSONB(value, output_field=self.output_field)
+
+        new_source_expressions.extend((Value(f"{{{path}}}"), value))
+        copy.set_source_expressions(new_source_expressions)
+        return super(JSONSet, copy).as_sql(compiler, connection, function="JSONB_SET", **extra_context)
+
+
+class JSONRemove(Func):
+    """
+    Unset and remove a single key in a JSONField.
+
+    Example:
+        model.objects.all().update(_custom_field_data=JSONRemove("_custom_field_data", "cf_key"))
+
+    Limitations:
+        - Postgres and MySQL only.
+        - Does *not* support nested lookups (`key1__key2`), only a single top-level key.
+        - Unlike the referenced Django PR, supports only a single key, not N keys.
+
+    References:
+        - https://code.djangoproject.com/ticket/32519
+        - https://github.com/django/django/pull/18489/files
+    """
+
+    def __init__(self, expression, path):
+        self.path = path
+        super().__init__(expression)
+
+    def as_sql(self, compiler, connection, function=None, **extra_context):
+        """
+        MySQL implementation based on https://github.com/django/django/pull/18489/files.
+
+        Creates a copy of this object with appropriately transformed self.path for MySQL JSON_REMOVE().
+        """
+        if connection.vendor != "mysql":
+            raise NotSupportedError(f"JSONSet is not implemented for database {connection.vendor}")
+
+        copy = self.copy()
+        new_source_expressions = copy.get_source_expressions()
+
+        new_source_expressions.append(Value(compile_json_path([self.path])))
+
+        copy.set_source_expressions(new_source_expressions)
+        return super(JSONRemove, copy).as_sql(compiler, connection, function="JSON_REMOVE", **extra_context)
+
+    def as_postgresql(self, compiler, connection, function=None, **extra_context):
+        """
+        PostgreSQL implementation based on https://github.com/django/django/pull/18489/files.
+
+        Creates a copy of this object with appropriately transformed self.path for Postgres `#-` operator.
+        """
+        copy = self.copy()
+        new_source_expressions = copy.get_source_expressions()
+
+        new_source_expressions.append(Value(f"{{{self.path}}}"))
+
+        copy.set_source_expressions(new_source_expressions)
+        return super(JSONRemove, copy).as_sql(
+            compiler, connection, template="%(expressions)s", arg_joiner="#- ", **extra_context
+        )
+
+
 class JSONBAgg(Aggregate):
     """
     Like django.contrib.postgres.aggregates.JSONBAgg, but different.

nautobot/core/tests/test_models_query_functions.py

@@ -0,0 +1,108 @@
+from nautobot.core.models.query_functions import JSONRemove, JSONSet
+from nautobot.core.testing import TestCase
+from nautobot.dcim.models import Manufacturer
+
+
+class JSONFuncTests(TestCase):
+    """Test JSONSet and JSONRemove functionality."""
+
+    def test_json_set(self):
+        # Setting a key/value should efficiently work
+        with self.assertNumQueries(1):
+            Manufacturer.objects.all().update(_custom_field_data=JSONSet("_custom_field_data", "a", 1))
+        for mfr in Manufacturer.objects.all():
+            self.assertIn("a", mfr._custom_field_data)
+            self.assertEqual(1, mfr._custom_field_data["a"])
+
+        # Setting a different key/value shouldn't overwrite other keys
+        with self.assertNumQueries(1):
+            Manufacturer.objects.all().update(_custom_field_data=JSONSet("_custom_field_data", "b", "text"))
+        for mfr in Manufacturer.objects.all():
+            self.assertIn("a", mfr._custom_field_data)
+            self.assertEqual(1, mfr._custom_field_data["a"])
+            self.assertIn("b", mfr._custom_field_data)
+            self.assertEqual("text", mfr._custom_field_data["b"])
+
+        # Setting a key/value again should overwrite that value only
+        with self.assertNumQueries(1):
+            Manufacturer.objects.all().update(_custom_field_data=JSONSet("_custom_field_data", "b", "more text"))
+        for mfr in Manufacturer.objects.all():
+            self.assertIn("a", mfr._custom_field_data)
+            self.assertEqual(1, mfr._custom_field_data["a"])
+            self.assertIn("b", mfr._custom_field_data)
+            self.assertEqual("more text", mfr._custom_field_data["b"])
+
+        # A filtered query should be updatable
+        with self.assertNumQueries(1):
+            Manufacturer.objects.filter(name__istartswith="a").update(
+                _custom_field_data=JSONSet("_custom_field_data", "a", None)
+            )
+        for mfr in Manufacturer.objects.filter(name__istartswith="a"):
+            self.assertIn("a", mfr._custom_field_data)
+            self.assertEqual(None, mfr._custom_field_data["a"])
+        for mfr in Manufacturer.objects.exclude(name__istartswith="a"):
+            self.assertIn("a", mfr._custom_field_data)
+            self.assertEqual(1, mfr._custom_field_data["a"])
+        for mfr in Manufacturer.objects.all():
+            self.assertIn("b", mfr._custom_field_data)
+            self.assertEqual("more text", mfr._custom_field_data["b"])
+
+        # Setting a value doesn't require all existing values to be homogeneous
+        with self.assertNumQueries(1):
+            Manufacturer.objects.all().update(_custom_field_data=JSONSet("_custom_field_data", "a", "hello"))
+        for mfr in Manufacturer.objects.all():
+            self.assertIn("a", mfr._custom_field_data)
+            self.assertEqual("hello", mfr._custom_field_data["a"])
+            self.assertIn("b", mfr._custom_field_data)
+            self.assertEqual("more text", mfr._custom_field_data["b"])
+
+    def test_json_remove(self):
+        Manufacturer.objects.all().update(_custom_field_data=JSONSet("_custom_field_data", "a", 1))
+        Manufacturer.objects.filter(name__istartswith="a").update(
+            _custom_field_data=JSONSet("_custom_field_data", "b", "hello")
+        )
+        Manufacturer.objects.exclude(name__istartswith="a").update(
+            _custom_field_data=JSONSet("_custom_field_data", "b", "world")
+        )
+
+        # Should be able to clear all values for a key without impacting other keys
+        with self.assertNumQueries(1):
+            Manufacturer.objects.all().update(_custom_field_data=JSONRemove("_custom_field_data", "a"))
+        for mfr in Manufacturer.objects.all():
+            self.assertNotIn("a", mfr._custom_field_data)
+            self.assertIn("b", mfr._custom_field_data)
+        for mfr in Manufacturer.objects.filter(name__istartswith="a"):
+            self.assertEqual("hello", mfr._custom_field_data["b"])
+        for mfr in Manufacturer.objects.exclude(name__istartswith="a"):
+            self.assertEqual("world", mfr._custom_field_data["b"])
+
+        # Clearing a value that doesn't exist should be safe
+        with self.assertNumQueries(1):
+            Manufacturer.objects.all().update(_custom_field_data=JSONRemove("_custom_field_data", "a"))
+        for mfr in Manufacturer.objects.all():
+            self.assertNotIn("a", mfr._custom_field_data)
+            self.assertIn("b", mfr._custom_field_data)
+        for mfr in Manufacturer.objects.filter(name__istartswith="a"):
+            self.assertEqual("hello", mfr._custom_field_data["b"])
+        for mfr in Manufacturer.objects.exclude(name__istartswith="a"):
+            self.assertEqual("world", mfr._custom_field_data["b"])
+
+        # Subsets should be updateable
+        with self.assertNumQueries(1):
+            Manufacturer.objects.filter(name__istartswith="a").update(
+                _custom_field_data=JSONRemove("_custom_field_data", "b")
+            )
+        for mfr in Manufacturer.objects.all():
+            self.assertNotIn("a", mfr._custom_field_data)
+        for mfr in Manufacturer.objects.filter(name__istartswith="a"):
+            self.assertNotIn("b", mfr._custom_field_data)
+        for mfr in Manufacturer.objects.exclude(name__istartswith="a"):
+            self.assertIn("b", mfr._custom_field_data)
+            self.assertEqual("world", mfr._custom_field_data["b"])
+
+        # Non-homogeneous data should be updatable
+        with self.assertNumQueries(1):
+            Manufacturer.objects.all().update(_custom_field_data=JSONRemove("_custom_field_data", "b"))
+        for mfr in Manufacturer.objects.all():
+            self.assertNotIn("a", mfr._custom_field_data)
+            self.assertNotIn("b", mfr._custom_field_data)

nautobot/dcim/templates/dcim/modulebay_create.html

@@ -0,0 +1,39 @@
+{% extends 'dcim/device_component_add.html' %}
+
+{% block javascript %}
+{{ block.super }}
+<script>
+document.addEventListener('DOMContentLoaded', function() {
+    var position_field = document.getElementById('id_position_pattern');
+    var source_arr = position_field.getAttribute('source').split(" ");
+    var length = position_field.getAttribute('maxlength');
+    position_field.setAttribute('_changed', Boolean(position_field.value))
+    position_field.addEventListener('change', function() {
+        position_field.setAttribute('_changed', Boolean(position_field.value))
+    });
+    function repopulate() {
+        let str = "";
+        for (source_str of source_arr) {
+            if (str != "") {
+                str += " ";
+            }
+            let source_id = 'id_' + source_str;
+            let source = document.getElementById(source_id)
+            str += source.value;
+        }
+        position_field.value = str.slice(0, length ? length : 255);
+    };
+    for (source_str of source_arr) {
+        let source_id = 'id_' + source_str;
+        let source = document.getElementById(source_id);
+        source.addEventListener('keyup', function() {
+            if (position_field && position_field.getAttribute('_changed')=="false") {
+                repopulate();
+            }
+        });
+    }
+    document.getElementsByClassName('reslugify')[0].addEventListener('click', repopulate);
+    document.getElementsByClassName('reslugify')[0].setAttribute('data-original-title', "Regenerate position");
+});
+</script>
+{% endblock javascript %}

nautobot/dcim/templates/dcim/modulebay_update.html

@@ -0,0 +1,39 @@
+{% extends 'generic/object_create.html' %}
+
+{% block javascript %}
+{{ block.super }}
+<script>
+document.addEventListener('DOMContentLoaded', function() {
+    var position_field = document.getElementById('id_position');
+    var source_arr = position_field.getAttribute('source').split(" ");
+    var length = position_field.getAttribute('maxlength');
+    position_field.setAttribute('_changed', Boolean(position_field.value))
+    position_field.addEventListener('change', function() {
+        position_field.setAttribute('_changed', Boolean(position_field.value))
+    });
+    function repopulate() {
+        let str = "";
+        for (source_str of source_arr) {
+            if (str != "") {
+                str += " ";
+            }
+            let source_id = 'id_' + source_str;
+            let source = document.getElementById(source_id)
+            str += source.value;
+        }
+        position_field.value = str.slice(0, length ? length : 255);
+    };
+    for (source_str of source_arr) {
+        let source_id = 'id_' + source_str;
+        let source = document.getElementById(source_id);
+        source.addEventListener('keyup', function() {
+            if (position_field && position_field.getAttribute('_changed')=="false") {
+                repopulate();
+            }
+        });
+    }
+    document.getElementsByClassName('reslugify')[0].addEventListener('click', repopulate);
+    document.getElementsByClassName('reslugify')[0].setAttribute('data-original-title', "Regenerate position");
+});
+</script>
+{% endblock javascript %}

nautobot/dcim/views.py

@@ -3241,7 +3241,7 @@ class ModuleBayUIViewSet(ModuleBayCommonViewSetMixin, NautobotUIViewSet):
     model_form_class = forms.ModuleBayForm
     serializer_class = serializers.ModuleBaySerializer
     table_class = tables.ModuleBayTable
-    create_template_name = "dcim/device_component_add.html"
+    create_template_name = "dcim/modulebay_create.html"
 
     def get_extra_context(self, request, instance):
         if instance:

nautobot/extras/api/customfields.py

@@ -40,15 +40,7 @@ class CustomFieldDefaultValues:
 class CustomFieldsDataField(Field):
     @property
     def custom_field_keys(self):
-        """
-        Cache CustomField keys assigned to this model to avoid redundant database queries
-        """
-        if not hasattr(self, "_custom_field_keys"):
-            content_type = ContentType.objects.get_for_model(self.parent.Meta.model)
-            self._custom_field_keys = CustomField.objects.filter(content_types=content_type).values_list(
-                "key", flat=True
-            )
-        return self._custom_field_keys
+        return CustomField.objects.keys_for_model(self.parent.Meta.model)
 
     def to_representation(self, obj):
         return {key: obj.get(key) for key in self.custom_field_keys}
@@ -58,7 +50,8 @@ class CustomFieldsDataField(Field):
 
         # Discard any entries in data that do not align with actual CustomFields - this matches the REST API behavior
         # for top-level serializer fields that do not exist or are not writable
-        data = {key: value for key, value in data.items() if key in self.custom_field_keys}
+        custom_field_keys = self.custom_field_keys
+        data = {key: value for key, value in data.items() if key in custom_field_keys}
 
         # If updating an existing instance, start with existing _custom_field_data
         if self.parent.instance:

nautobot/extras/context_managers.py

@@ -204,14 +204,34 @@ def web_request_context(
             yield request
     finally:
         jobs_reloaded = False
+        # In bulk operations, we are performing the same action (create/update/delete) on the same content-type.
+        # Save some repeated database queries by reusing the same evaluated querysets where applicable:
+        jobhook_queryset = None
+        webhook_queryset = None
+        last_action = None
+        last_content_type = None
         # enqueue jobhooks and webhooks, use change_context.change_id in case change_id was not supplied
         for object_change in (
-            ObjectChange.objects.filter(request_id=change_context.change_id).order_by("time").iterator()
+            ObjectChange.objects.select_related("changed_object_type", "user")
+            .filter(request_id=change_context.change_id)
+            .order_by("time")  # default ordering is -time but we want oldest first not newest first
+            .iterator()
         ):
+            if object_change.action != last_action or object_change.changed_object_type != last_content_type:
+                jobhook_queryset = None
+                webhook_queryset = None
+
             if context != ObjectChangeEventContextChoices.CONTEXT_JOB_HOOK:
                 # Make sure JobHooks are up to date (only once) before calling them
-                jobs_reloaded |= enqueue_job_hooks(object_change, may_reload_jobs=(not jobs_reloaded))
-            enqueue_webhooks(object_change)
+                did_reload_jobs, jobhook_queryset = enqueue_job_hooks(
+                    object_change, may_reload_jobs=(not jobs_reloaded), jobhook_queryset=jobhook_queryset
+                )
+                if did_reload_jobs:
+                    jobs_reloaded = True
+
+            webhook_queryset = enqueue_webhooks(object_change, webhook_queryset=webhook_queryset)
+            last_action = object_change.action
+            last_content_type = object_change.changed_object_type
 
 
 @contextmanager

nautobot/extras/jobs.py

@@ -1144,41 +1144,47 @@ def run_job(self, job_class_path, *args, **kwargs):
         raise
 
 
-def enqueue_job_hooks(object_change, may_reload_jobs=True):
+def enqueue_job_hooks(object_change, may_reload_jobs=True, jobhook_queryset=None):
     """
     Find job hook(s) assigned to this changed object type + action and enqueue them to be processed.
 
+    Args:
+        object_change (ObjectChange): The change that may trigger JobHooks to execute.
+        may_reload_jobs (bool): Whether to reload JobHook source code from disk to guarantee up-to-date code.
+        jobhook_queryset (QuerySet): Previously retrieved set of JobHooks to potentially enqueue
+
     Returns:
-        jobs_reloaded (bool): whether Jobs were reloaded to make this happen
+        result (tuple[bool, QuerySet]): whether Jobs were reloaded here, and the jobhooks that were considered
     """
     jobs_reloaded = False
 
     # Job hooks cannot trigger other job hooks
     if object_change.change_context == ObjectChangeEventContextChoices.CONTEXT_JOB_HOOK:
-        return jobs_reloaded
+        return jobs_reloaded, jobhook_queryset
 
     # Determine whether this type of object supports job hooks
     content_type = object_change.changed_object_type
     if content_type not in change_logged_models_queryset():
-        return jobs_reloaded
+        return jobs_reloaded, jobhook_queryset
 
     # Retrieve any applicable job hooks
-    action_flag = {
-        ObjectChangeActionChoices.ACTION_CREATE: "type_create",
-        ObjectChangeActionChoices.ACTION_UPDATE: "type_update",
-        ObjectChangeActionChoices.ACTION_DELETE: "type_delete",
-    }[object_change.action]
-    job_hooks = JobHook.objects.filter(content_types=content_type, enabled=True, **{action_flag: True})
+    if jobhook_queryset is None:
+        action_flag = {
+            ObjectChangeActionChoices.ACTION_CREATE: "type_create",
+            ObjectChangeActionChoices.ACTION_UPDATE: "type_update",
+            ObjectChangeActionChoices.ACTION_DELETE: "type_delete",
+        }[object_change.action]
+        jobhook_queryset = JobHook.objects.filter(content_types=content_type, enabled=True, **{action_flag: True})
 
-    if not job_hooks.exists():
-        return jobs_reloaded
+    if not jobhook_queryset:  # not .exists() as we *want* to populate the queryset cache
+        return jobs_reloaded, jobhook_queryset
 
     # Enqueue the jobs related to the job_hooks
     if may_reload_jobs:
         get_jobs(reload=True)
         jobs_reloaded = True
 
-    for job_hook in job_hooks:
+    for job_hook in jobhook_queryset:
         job_model = job_hook.job
         if not job_model.installed or not job_model.enabled:
             logger.warning(
@@ -1189,4 +1195,4 @@ def enqueue_job_hooks(object_change, may_reload_jobs=True):
         else:
             JobResult.enqueue_job(job_model, object_change.user, object_change=object_change.pk)
 
-    return jobs_reloaded
+    return jobs_reloaded, jobhook_queryset

nautobot/extras/models/customfields.py

@@ -391,6 +391,18 @@ class CustomFieldManager(BaseManager.from_queryset(RestrictedQuerySet)):
 
     get_for_model.cache_key_prefix = "nautobot.extras.customfield.get_for_model"
 
+    def keys_for_model(self, model):
+        """Return list of all keys for CustomFields assigned to the given model."""
+        concrete_model = model._meta.concrete_model
+        cache_key = f"{self.keys_for_model.cache_key_prefix}.{concrete_model._meta.label_lower}"
+        keys = cache.get(cache_key)
+        if keys is None:
+            keys = list(self.get_for_model(model).values_list("key", flat=True))
+            cache.set(cache_key, keys)
+        return keys
+
+    keys_for_model.cache_key_prefix = "nautobot.extras.customfield.keys_for_model"
+
 
 @extras_features("webhooks")
 class CustomField(

nautobot/extras/signals.py

@@ -91,6 +91,8 @@ def invalidate_models_cache(sender, **kwargs):
     with contextlib.suppress(redis.exceptions.ConnectionError):
         # TODO: *maybe* target more narrowly, e.g. only clear the cache for specific related content-types?
         cache.delete_pattern(f"{manager.get_for_model.cache_key_prefix}.*")
+        if hasattr(manager, "keys_for_model"):
+            cache.delete_pattern(f"{manager.keys_for_model.cache_key_prefix}.*")
 
 
 @receiver(post_save, sender=Relationship)