nautobot 2.4.0b1__py3-none-any.whl → 2.4.2__py3-none-any.whl

nautobot/core/testing/api.py

@@ -5,11 +5,13 @@ from typing import Optional, Sequence, Union
 from django.conf import settings
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
+from django.db import connections, DEFAULT_DB_ALIAS
 from django.db.models import ForeignKey, ManyToManyField, QuerySet
 from django.test import override_settings, tag
+from django.test.utils import CaptureQueriesContext
 from django.urls import reverse
 from django.utils.text import slugify
-from rest_framework import status
+from rest_framework import serializers, status
 from rest_framework.relations import ManyRelatedField
 from rest_framework.test import APITransactionTestCase as _APITransactionTestCase
 
@@ -94,6 +96,15 @@ class APITestCase(views.ModelTestCase):
         for verboten in self.VERBOTEN_STRINGS:
             self.assertNotIn(verboten, response_raw_content)
 
+    def get_m2m_fields(self):
+        """Get serializer field names that are many-to-many or one-to-many and thus affected by ?exclude_m2m=true."""
+        serializer_class = get_serializer_for_model(self.model)
+        m2m_fields = []
+        for field_name, field_instance in serializer_class().fields.items():
+            if isinstance(field_instance, (serializers.ManyRelatedField, serializers.ListSerializer)):
+                m2m_fields.append(field_name)
+        return m2m_fields
+
 
 @tag("unit")
 class APIViewTestCases:
@@ -231,7 +242,7 @@ class APIViewTestCases:
         def get_depth_fields(self):
             """Get a list of model fields that could be tested with the ?depth query parameter"""
             depth_fields = []
-            for field in self.model._meta.fields:
+            for field in self.model._meta.get_fields():
                 if not field.name.startswith("_"):
                     if isinstance(field, (ForeignKey, GenericForeignKey, ManyToManyField, core_fields.TagsField)) and (
                         # we represent content-types as "app_label.modelname" rather than as FKs
@@ -240,6 +251,9 @@ class APIViewTestCases:
                         and not (field.name == "user" and self.model == users_models.Token)
                     ):
                         depth_fields.append(field.name)
+            serializer_class = get_serializer_for_model(self.model)
+            serializer = serializer_class()
+            depth_fields = [field_name for field_name in depth_fields if field_name in serializer.fields]
             return depth_fields
 
         @override_settings(EXEMPT_VIEW_PERMISSIONS=["*"])
@@ -269,9 +283,12 @@ class APIViewTestCases:
             GET a list of objects using the "?depth=0" parameter.
             """
             depth_fields = self.get_depth_fields()
+            m2m_fields = self.get_m2m_fields()
             self.add_permissions(f"{self.model._meta.app_label}.view_{self.model._meta.model_name}")
-            url = f"{self._get_list_url()}?depth=0"
-            response = self.client.get(url, **self.header)
+            list_url = f"{self._get_list_url()}?depth=0"
+            with CaptureQueriesContext(connections[DEFAULT_DB_ALIAS]) as cqc:
+                response = self.client.get(list_url, **self.header)
+            base_num_queries = len(cqc)
 
             self.assertHttpStatus(response, status.HTTP_200_OK)
             self.assertIsInstance(response.data, dict)
@@ -280,15 +297,23 @@ class APIViewTestCases:
             self.assert_no_verboten_content(response)
 
             for response_data in response.data["results"]:
+                for field in m2m_fields:
+                    self.assertIn(field, response_data)
+                    self.assertIsInstance(response_data[field], list)
                 for field in depth_fields:
                     self.assertIn(field, response_data)
                     if isinstance(response_data[field], list):
                         for entry in response_data[field]:
                             self.assertIsInstance(entry, dict)
-                            self.assertTrue(is_uuid(entry["id"]))
+                            if entry["object_type"] in ["auth.group"]:
+                                self.assertIsInstance(entry["id"], int)
+                            else:
+                                self.assertTrue(is_uuid(entry["id"]))
+                            self.assertEqual(len(entry.keys()), 3)  # just id/object_type/url
                     else:
                         if response_data[field] is not None:
                             self.assertIsInstance(response_data[field], dict)
+                            self.assertEqual(len(response_data[field].keys()), 3)  # just id/object_type/url
                             url = response_data[field]["url"]
                             pk = response_data[field]["id"]
                             object_type = response_data[field]["object_type"]
@@ -296,22 +321,67 @@ class APIViewTestCases:
                             # URL ending in the UUID of the relevant object:
                             # http://nautobot.example.com/api/circuits/providers/<uuid>/
                             #                                                    ^^^^^^
-                            self.assertTrue(is_uuid(url.split("/")[-2]))
-                            self.assertTrue(is_uuid(pk))
+                            if object_type in ["auth.group"]:
+                                self.assertIsInstance(url.split("/")[-2], int)
+                                self.assertIsInstance(pk, int)
+                            else:
+                                self.assertTrue(is_uuid(url.split("/")[-2]))
+                                self.assertTrue(is_uuid(pk))
 
                             with self.subTest(f"Assert object_type {object_type} is valid"):
                                 app_label, model_name = object_type.split(".")
                                 ContentType.objects.get(app_label=app_label, model=model_name)
 
+            list_url += "&exclude_m2m=true"
+            with CaptureQueriesContext(connections[DEFAULT_DB_ALIAS]) as cqc:
+                response = self.client.get(list_url, **self.header)
+
+            self.assertHttpStatus(response, status.HTTP_200_OK)
+            self.assertIsInstance(response.data, dict)
+            self.assertIn("results", response.data)
+            self.assert_no_verboten_content(response)
+
+            if m2m_fields:
+                if self.model._meta.app_label in [
+                    "circuits",
+                    "cloud",
+                    "dcim",
+                    "extras",
+                    "ipam",
+                    "tenancy",
+                    "users",
+                    "virtualization",
+                    "wireless",
+                ]:
+                    self.assertLess(
+                        len(cqc), base_num_queries, "Number of queries did not decrease with ?exclude_m2m=true"
+                    )
+                else:
+                    # Less strict check for non-core APIs
+                    self.assertLessEqual(
+                        len(cqc), base_num_queries, "Number of queries increased with ?exclude_m2m=true"
+                    )
+            else:
+                # No M2M fields to exclude
+                self.assertLessEqual(len(cqc), base_num_queries, "Number of queries increased with ?exclude_m2m=true")
+
+            for response_data in response.data["results"]:
+                for field in m2m_fields:
+                    self.assertNotIn(field, response_data)
+                # TODO: we should assert that all other fields are still present, but there's a few corner cases...
+
         @override_settings(EXEMPT_VIEW_PERMISSIONS=[])
         def test_list_objects_depth_1(self):
             """
             GET a list of objects using the "?depth=1" parameter.
             """
             depth_fields = self.get_depth_fields()
+            m2m_fields = self.get_m2m_fields()
             self.add_permissions(f"{self.model._meta.app_label}.view_{self.model._meta.model_name}")
-            url = f"{self._get_list_url()}?depth=1"
-            response = self.client.get(url, **self.header)
+            list_url = f"{self._get_list_url()}?depth=1"
+            with CaptureQueriesContext(connections[DEFAULT_DB_ALIAS]) as cqc:
+                response = self.client.get(list_url, **self.header)
+            base_num_queries = len(cqc)
 
             self.assertHttpStatus(response, status.HTTP_200_OK)
             self.assertIsInstance(response.data, dict)
@@ -320,16 +390,65 @@ class APIViewTestCases:
             self.assert_no_verboten_content(response)
 
             for response_data in response.data["results"]:
+                for field in m2m_fields:
+                    self.assertIn(field, response_data)
+                    self.assertIsInstance(response_data[field], list)
                 for field in depth_fields:
                     self.assertIn(field, response_data)
                     if isinstance(response_data[field], list):
                         for entry in response_data[field]:
                             self.assertIsInstance(entry, dict)
-                            self.assertTrue(is_uuid(entry["id"]))
+                            if entry["object_type"] in ["auth.group"]:
+                                self.assertIsInstance(entry["id"], int)
+                            else:
+                                self.assertTrue(is_uuid(entry["id"]))
+                            self.assertGreater(len(entry.keys()), 3, entry)  # not just id/object_type/url!
                     else:
                         if response_data[field] is not None:
                             self.assertIsInstance(response_data[field], dict)
-                            self.assertTrue(is_uuid(response_data[field]["id"]))
+                            if response_data[field]["object_type"] in ["auth.group"]:
+                                self.assertIsInstance(response_data[field]["id"], int)
+                            else:
+                                self.assertTrue(is_uuid(response_data[field]["id"]))
+                            self.assertGreater(len(response_data[field].keys()), 3, response_data[field])
+
+            list_url += "&exclude_m2m=true"
+            with CaptureQueriesContext(connections[DEFAULT_DB_ALIAS]) as cqc:
+                response = self.client.get(list_url, **self.header)
+
+            self.assertHttpStatus(response, status.HTTP_200_OK)
+            self.assertIsInstance(response.data, dict)
+            self.assertIn("results", response.data)
+            self.assert_no_verboten_content(response)
+
+            if m2m_fields:
+                if self.model._meta.app_label in [
+                    "circuits",
+                    "cloud",
+                    "dcim",
+                    "extras",
+                    "ipam",
+                    "tenancy",
+                    "users",
+                    "virtualization",
+                    "wireless",
+                ]:
+                    self.assertLess(
+                        len(cqc), base_num_queries, "Number of queries did not decrease with ?exclude_m2m=true"
+                    )
+                else:
+                    # Less strict check for non-core APIs
+                    self.assertLessEqual(
+                        len(cqc), base_num_queries, "Number of queries increased with ?exclude_m2m=true"
+                    )
+            else:
+                # No M2M fields to exclude
+                self.assertLessEqual(len(cqc), base_num_queries, "Number of queries increased with ?exclude_m2m=true")
+
+            for response_data in response.data["results"]:
+                for field in m2m_fields:
+                    self.assertNotIn(field, response_data)
+                # TODO: we should assert that all other fields are still present, but there's a few corner cases...
 
         @override_settings(EXEMPT_VIEW_PERMISSIONS=[])
         def test_list_objects_without_permission(self):
@@ -655,7 +774,11 @@ class APIViewTestCases:
             self.assertHttpStatus(response, status.HTTP_201_CREATED, csv_data)
             # Note that create via CSV is always treated as a bulk-create, and so the response is always a list of dicts
             new_instance = self._get_queryset().get(pk=response.data[0]["id"])
-            self.assertNotEqual(new_instance.pk, orig_pk)
+            if isinstance(orig_pk, int):
+                self.assertNotEqual(new_instance.pk, orig_pk)
+            else:
+                # for our non-integer PKs, we're expecting the creation to respect the requested PK
+                self.assertEqual(new_instance.pk, orig_pk)
 
             new_serializer = serializer_class(new_instance, context={"request": None})
             new_data = new_serializer.data
@@ -737,7 +860,7 @@ class APIViewTestCases:
 
             def strip_serialized_object(this_object):
                 """
-                Only here to work around acceptable differences in PATCH response vs GET response which are known bugs.
+                Work around acceptable differences in PATCH response vs GET response which are known behaviors.
                 """
                 # Work around for https://github.com/nautobot/nautobot/issues/3321
                 this_object.pop("last_updated", None)
@@ -746,6 +869,12 @@ class APIViewTestCases:
                 this_object.pop("config_context", None)
                 this_object.pop("relationships", None)
 
+                serializer = get_serializer_for_model(self.model)()
+                for field_name, field_instance in serializer.fields.items():
+                    if field_instance.read_only:
+                        # Likely a derived field, might change as a consequence of other data updates
+                        this_object.pop(field_name, None)
+
                 for value in this_object.values():
                     if isinstance(value, dict):
                         strip_serialized_object(value)
@@ -789,7 +918,7 @@ class APIViewTestCases:
             self.assertEqual(initial_serialized_object, serialized_object)
 
             # Verify ObjectChange creation -- yes, even though nothing actually changed
-            # This may change (hah) at some point -- see https://github.com/nautobot/nautobot/issues/3321
+            # TODO: This may change (hah) at some point -- see https://github.com/nautobot/nautobot/issues/3321
             if hasattr(self.model, "to_objectchange"):
                 objectchanges = lookup.get_changes_for_model(instance)
                 self.assertEqual(objectchanges[0].action, extras_choices.ObjectChangeActionChoices.ACTION_UPDATE)
@@ -798,10 +927,16 @@ class APIViewTestCases:
             # Verify that a PATCH with some data updates that data correctly.
             response = self.client.patch(url, update_data, format="json", **self.header)
             self.assertHttpStatus(response, status.HTTP_200_OK)
+            serialized_object = response.json()
+            strip_serialized_object(serialized_object)
             # Check for unexpected side effects on fields we DIDN'T intend to update
             for field in initial_serialized_object:
                 if field not in update_data:
-                    self.assertEqual(initial_serialized_object[field], serialized_object[field])
+                    self.assertEqual(
+                        initial_serialized_object[field],
+                        serialized_object[field],
+                        f"data changed unexpectedly for field '{field}'",
+                    )
             instance.refresh_from_db()
             self.assertInstanceEqual(instance, update_data, exclude=self.validation_excluded_fields, api=True)
 
@@ -810,6 +945,26 @@ class APIViewTestCases:
                 objectchanges = lookup.get_changes_for_model(instance)
                 self.assertEqual(objectchanges[0].action, extras_choices.ObjectChangeActionChoices.ACTION_UPDATE)
 
+            # Verify that a PATCH with ?exclude_m2m=true correctly excludes many-to-many fields from the response
+            # This also doubles as a test for idempotence of the PATCH request.
+            response = self.client.patch(url + "?exclude_m2m=true", update_data, format="json", **self.header)
+            self.assertHttpStatus(response, status.HTTP_200_OK)
+            m2m_fields = self.get_m2m_fields()
+            serialized_object = response.json()
+            strip_serialized_object(serialized_object)
+            for field in m2m_fields:
+                self.assertNotIn(field, serialized_object)
+            # Check for unexpected side effects on fields we DIDN'T intend to update
+            for field in initial_serialized_object:
+                if field not in update_data and field not in m2m_fields:
+                    self.assertEqual(
+                        initial_serialized_object[field],
+                        serialized_object[field],
+                        f"data changed unexpectedly for field '{field}'",
+                    )
+            instance.refresh_from_db()
+            self.assertInstanceEqual(instance, update_data, exclude=self.validation_excluded_fields, api=True)
+
         def test_get_put_round_trip(self):
             """GET and then PUT an object and verify that it's accepted and unchanged."""
             self.maxDiff = None

nautobot/core/testing/filters.py

@@ -1,6 +1,6 @@
 import random
 import string
-from typing import ClassVar, Iterable
+from typing import ClassVar, Iterable, Optional
 
 from django.contrib.contenttypes.models import ContentType
 from django.db.models import Count, Q, QuerySet
@@ -30,6 +30,8 @@ class FilterTestCases:
     class BaseFilterTestCase(views.TestCase):
         """Base class for testing of FilterSets."""
 
+        queryset: ClassVar[Optional[QuerySet]] = None  # TODO: declared as Optional only to avoid a breaking change
+
         def get_filterset_test_values(self, field_name, queryset=None):
             """Returns a list of distinct values from the requested queryset field to use in filterset tests.
 
@@ -71,8 +73,7 @@ class FilterTestCases:
     class FilterTestCase(BaseFilterTestCase):
         """Add common tests for all FilterSets."""
 
-        queryset: ClassVar[QuerySet]
-        filterset: ClassVar[FilterSet]
+        filterset: ClassVar[Optional[type[FilterSet]]] = None  # TODO: declared Optional only to avoid breaking change
 
         # filter predicate fields that should be excluded from q test case
         exclude_q_filter_predicates = []
@@ -84,7 +85,7 @@ class FilterTestCases:
         #       ["filter1"],
         #       ["filter2", "field2__name"],
         #   ]
-        generic_filter_tests: ClassVar[Iterable]
+        generic_filter_tests: ClassVar[Iterable] = ()
 
         def setUp(self):
             for attr in ["queryset", "filterset", "generic_filter_tests"]:
@@ -94,12 +95,38 @@ class FilterTestCases:
 
         def get_q_filter(self):
             """Helper method to return q filter."""
+            self.assertIsNotNone(self.filterset)
             return self.filterset.declared_filters["q"].filter_predicates
 
+        def test_id(self):
+            """Verify that the filterset supports filtering by id with only lookup `__n`."""
+            self.assertIsNotNone(self.filterset)
+
+            with self.subTest("Assert `id`"):
+                params = {"id": list(self.queryset.values_list("pk", flat=True)[:2])}
+                expected_queryset = self.queryset.filter(id__in=params["id"])
+                filterset = self.filterset(params, self.queryset)  # pylint: disable=not-callable  # see assertion above
+                self.assertTrue(filterset.is_valid())
+                self.assertQuerysetEqualAndNotEmpty(filterset.qs.order_by("id"), expected_queryset.order_by("id"))
+
+            with self.subTest("Assert negate lookup"):
+                params = {"id__n": list(self.queryset.values_list("pk", flat=True)[:2])}
+                expected_queryset = self.queryset.exclude(id__in=params["id__n"])
+                filterset = self.filterset(params, self.queryset)  # pylint: disable=not-callable  # see assertion above
+                self.assertTrue(filterset.is_valid())
+                self.assertQuerysetEqualAndNotEmpty(filterset.qs.order_by("id"), expected_queryset.order_by("id"))
+
+            with self.subTest("Assert invalid lookup"):
+                params = {"id__in": list(self.queryset.values_list("pk", flat=True)[:2])}
+                filterset = self.filterset(params, self.queryset)  # pylint: disable=not-callable  # see assertion above
+                self.assertFalse(filterset.is_valid())
+                self.assertIn("Unknown filter field", filterset.errors.as_text())
+
         def test_invalid_filter(self):
             """Verify that the filterset reports as invalid when initialized with an unsupported filter parameter."""
             params = {"ice_cream_flavor": ["chocolate"]}
-            self.assertFalse(self.filterset(params, self.queryset).is_valid())
+            self.assertIsNotNone(self.filterset)
+            self.assertFalse(self.filterset(params, self.queryset).is_valid())  # pylint: disable=not-callable
 
         def test_filters_generic(self):
             """Test all multiple choice filters declared in `self.generic_filter_tests`.
@@ -167,13 +194,16 @@ class FilterTestCases:
                         status=Status.objects.get_for_model(ContactAssociation).last(),
                     )
 
+            if self.generic_filter_tests:
+                self.assertIsNotNone(self.filterset)
+
             for test in self.generic_filter_tests:
                 filter_name = test[0]
                 field_name = test[-1]  # default to filter_name if a second list item was not supplied
                 with self.subTest(f"{self.filterset.__name__} filter {filter_name} ({field_name})"):
                     test_data = self.get_filterset_test_values(field_name)
                     params = {filter_name: test_data}
-                    filterset_result = self.filterset(params, self.queryset).qs
+                    filterset_result = self.filterset(params, self.queryset).qs  # pylint: disable=not-callable
                     qs_result = self.queryset.filter(**{f"{field_name}__in": test_data}).distinct()
                     self.assertQuerysetEqualAndNotEmpty(filterset_result, qs_result, ordered=False)
 
@@ -184,6 +214,7 @@ class FilterTestCases:
             This test asserts that `filter=True` matches `self.queryset.filter(field__isnull=False)` and
             that `filter=False` matches `self.queryset.filter(field__isnull=True)`.
             """
+            self.assertIsNotNone(self.filterset)
             for filter_name, filter_object in self.filterset.get_filters().items():
                 if not isinstance(filter_object, RelatedMembershipBooleanFilter):
                     continue
@@ -191,11 +222,11 @@ class FilterTestCases:
                     continue
                 field_name = filter_object.field_name
                 with self.subTest(f"{self.filterset.__name__} RelatedMembershipBooleanFilter {filter_name} (True)"):
-                    filterset_result = self.filterset({filter_name: True}, self.queryset).qs
+                    filterset_result = self.filterset({filter_name: True}, self.queryset).qs  # pylint: disable=not-callable
                     qs_result = self.queryset.filter(**{f"{field_name}__isnull": filter_object.exclude}).distinct()
                     self.assertQuerysetEqualAndNotEmpty(filterset_result, qs_result)
                 with self.subTest(f"{self.filterset.__name__} RelatedMembershipBooleanFilter {filter_name} (False)"):
-                    filterset_result = self.filterset({filter_name: False}, self.queryset).qs
+                    filterset_result = self.filterset({filter_name: False}, self.queryset).qs  # pylint: disable=not-callable
                     qs_result = self.queryset.exclude(**{f"{field_name}__isnull": filter_object.exclude}).distinct()
                     self.assertQuerysetEqualAndNotEmpty(filterset_result, qs_result)
 
@@ -204,6 +235,8 @@ class FilterTestCases:
             if not issubclass(self.queryset.model, PrimaryModel):
                 self.skipTest("Not a PrimaryModel")
 
+            self.assertIsNotNone(self.filterset)
+
             # Find an instance with at least two tags (should be common given our factory design)
             for instance in list(self.queryset):
                 if len(instance.tags.all()) >= 2:
@@ -220,7 +253,7 @@ class FilterTestCases:
                 self.queryset.first().tags.add(test_tags_filter_a, test_tags_filter_b)
                 tags = [test_tags_filter_a, test_tags_filter_b]
             params = {"tags": [tags[0].name, tags[1].pk]}
-            filterset_result = self.filterset(params, self.queryset).qs
+            filterset_result = self.filterset(params, self.queryset).qs  # pylint: disable=not-callable
             # Tags is an AND filter not an OR filter
             qs_result = self.queryset.filter(tags=tags[0]).filter(tags=tags[1]).distinct()
             self.assertQuerysetEqualAndNotEmpty(filterset_result, qs_result)
@@ -271,6 +304,8 @@ class FilterTestCases:
             """
             self._assert_valid_filter_predicates(obj, obj_field_name)
 
+            self.assertIsNotNone(self.filterset)
+
             # Generic test only supports CharField or TextFields, skip all other types
             obj_field = obj._meta.get_field(obj_field_name)
             if not isinstance(obj_field, (CharField, TextField)):
@@ -290,7 +325,7 @@ class FilterTestCases:
                 lookup = randomized_attr_value[1:].upper()
                 model_queryset = self.queryset.filter(**{f"{filter_field_name}__icontains": lookup})
             params = {"q": lookup}
-            filterset_result = self.filterset(params, self.queryset)
+            filterset_result = self.filterset(params, self.queryset)  # pylint: disable=not-callable
 
             self.assertTrue(filterset_result.is_valid())
             self.assertQuerysetEqualAndNotEmpty(

nautobot/core/testing/forms.py

@@ -1,5 +1,6 @@
 from django.test import tag, TestCase
 
+from nautobot.core.api.constants import NON_FILTER_QUERY_PARAMS
 from nautobot.core.forms.fields import DynamicModelChoiceMixin
 from nautobot.core.utils.lookup import get_filterset_for_model
 
@@ -18,6 +19,7 @@ class FormTestCases:
                     continue
                 with self.subTest(f"Assert {self.form_class.__name__}.{field_name} query_params are valid."):
                     query_params_fields = set(field_class.query_params.keys())
+                    query_params_fields = query_params_fields - set(NON_FILTER_QUERY_PARAMS)
                     if not query_params_fields:
                         self.skipTest(f"{self.form_class.__name__}.{field_name} has no query_params")
                     field_model = field_class.queryset.model