nautobot 2.2.9__py3-none-any.whl → 2.3.0__py3-none-any.whl

nautobot/core/filters.py

@@ -1,4 +1,3 @@
-from collections import OrderedDict
 from copy import deepcopy
 import logging
 import uuid
@@ -125,8 +124,8 @@ class MultiValueUUIDFilter(django_filters.UUIDFilter, django_filters.MultipleCho
 
 class RelatedMembershipBooleanFilter(django_filters.BooleanFilter):
     """
-    BooleanFilter for related objects that will explicitly perform `exclude=True` and `isnull`
-    lookups. The `field_name` argument is required and must be set to the related field on the
+    BooleanFilter for related objects that will explicitly perform `isnull` lookups.
+    The `field_name` argument is required and must be set to the related field on the
     model.
 
     This should be used instead of a default `BooleanFilter` paired `method=`
@@ -134,15 +133,33 @@ class RelatedMembershipBooleanFilter(django_filters.BooleanFilter):
 
     Example:
 
-        has_interfaces = RelatedMembershipBooleanFilter(
-            field_name="interfaces",
-            label="Has interfaces",
+        has_modules = RelatedMembershipBooleanFilter(
+            field_name="module_bays__installed_module",
+            label="Has modules",
         )
+
+        This would generate a filter that returns instances that have at least one module
+        bay with an installed module. The `has_modules=False` filter would exclude instances
+        with at least one module bay with an installed module.
+
+        Set `exclude=True` to reverse the behavior of the filter. This __may__ be useful
+        for filtering on null directly related fields but this filter is not smart enough
+        to differentiate between `fieldA__fieldB__isnull` and `fieldA__isnull` so it's not
+        suitable for cases like the `has_empty_module_bays` filter where an instance may
+        not have any module bays.
+
+        See the below table for more information:
+
+        | value       | exclude          | Result
+        |-------------|------------------|-------
+        | True        | False (default)  | Return instances with at least one non-null match -- qs.filter(field_name__isnull=False)
+        | False       | False (default)  | Exclude instances with at least one non-null match -- qs.exclude(field_name__isnull=False)
+        | True        | True             | Return instances with at least one null match -- qs.filter(field_name__isnull=True)
+        | False       | True             | Exclude instances with at least one null match -- qs.exclude(field_name__isnull=True)
+
     """
 
-    def __init__(
-        self, field_name=None, lookup_expr="isnull", *, label=None, method=None, distinct=False, exclude=True, **kwargs
-    ):
+    def __init__(self, field_name=None, lookup_expr="isnull", *, label=None, method=None, distinct=True, **kwargs):
         if field_name is None:
             raise ValueError(f"Field name is required for {self.__class__.__name__}")
 
@@ -152,11 +169,23 @@ class RelatedMembershipBooleanFilter(django_filters.BooleanFilter):
             label=label,
             method=method,
             distinct=distinct,
-            exclude=exclude,
             widget=forms.StaticSelect2(choices=forms.BOOLEAN_CHOICES),
             **kwargs,
         )
 
+    def filter(self, qs, value):
+        if value in EMPTY_VALUES:
+            return qs
+        if self.distinct:
+            qs = qs.distinct()
+        lookup = f"{self.field_name}__{self.lookup_expr}"
+        if bool(value):
+            # if self.exclude=False, return instances with field populated
+            return qs.filter(**{lookup: self.exclude})
+        else:
+            # if self.exclude=False, exclude instances with field populated
+            return qs.exclude(**{lookup: self.exclude})
+
 
 class NumericArrayFilter(django_filters.NumberFilter):
     """
@@ -403,12 +432,22 @@ class NaturalKeyOrPKMultipleChoiceFilter(django_filters.ModelMultipleChoiceFilte
     Filter that supports filtering on values matching the `pk` field and another
     field of a foreign-key related object. The desired field is set using the `to_field_name`
     keyword argument on filter initialization (defaults to `name`).
+
+    NOTE that the `to_field_name` field does not have to be a "true" natural key (ie. unique), it
+    was just the best name we could come up with for this filter
+
     """
 
     field_class = forms.MultiMatchModelMultipleChoiceField
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, prefers_id=False, **kwargs):
+        """Initialize the NaturalKeyOrPKMultipleChoiceFilter.
+
+        Args:
+            prefers_id (bool, optional): Prefer PK (ID) over the 'to_field_name'. Defaults to False.
+        """
         self.natural_key = kwargs.setdefault("to_field_name", "name")
+        self.prefers_id = prefers_id
         super().__init__(*args, **kwargs)
 
     def get_filter_predicate(self, v):
@@ -725,8 +764,8 @@ class BaseFilterSet(django_filters.FilterSet):
     def get_fields(cls):
         fields = super().get_fields()
         if "id" not in fields and (cls._meta.exclude is None or "id" not in cls._meta.exclude):
-            # Add "id" as the first key in the `fields` OrderedDict
-            fields = OrderedDict(id=[django_filters.conf.settings.DEFAULT_LOOKUP_EXPR], **fields)
+            # Add "id" as the first key in the `fields` dict
+            fields = {"id": [django_filters.conf.settings.DEFAULT_LOOKUP_EXPR], **fields}
         return fields
 
     @classmethod
@@ -741,8 +780,51 @@ class BaseFilterSet(django_filters.FilterSet):
             if filter_name.startswith("_"):
                 del filters[filter_name]
 
+        if getattr(cls._meta.model, "is_contact_associable_model", False):
+            # Add "contacts" and "teams" filters
+            from nautobot.extras.models import Contact, Team
+
+            if "contacts" not in filters:
+                filters["contacts"] = NaturalKeyOrPKMultipleChoiceFilter(
+                    queryset=Contact.objects.all(),
+                    field_name="associated_contacts__contact",
+                    to_field_name="name",
+                    label="Contacts (name or ID)",
+                )
+
+            if "teams" not in filters:
+                filters["teams"] = NaturalKeyOrPKMultipleChoiceFilter(
+                    queryset=Team.objects.all(),
+                    field_name="associated_contacts__team",
+                    to_field_name="name",
+                    label="Teams (name or ID)",
+                )
+
+        if "dynamic_groups" not in filters and getattr(cls._meta.model, "is_dynamic_group_associable_model", False):
+            if not hasattr(cls._meta.model, "static_group_association_set"):
+                logger.warning(
+                    "Model %s has 'is_dynamic_group_associable_model = True' but lacks "
+                    "a 'static_group_association_set' attribute. Perhaps this is due to it inheriting from "
+                    "the deprecated DynamicGroupMixin class instead of the preferred DynamicGroupsModelMixin?",
+                    cls._meta.model,
+                )
+            else:
+                # Add "dynamic_groups" field as the last key
+                from nautobot.extras.models import DynamicGroup
+
+                filters["dynamic_groups"] = NaturalKeyOrPKMultipleChoiceFilter(
+                    queryset=DynamicGroup.objects.all(),
+                    field_name="static_group_association_set__dynamic_group",
+                    to_field_name="name",
+                    query_params={"content_type": cls._meta.model._meta.label_lower},
+                    label="Dynamic groups (name or ID)",
+                )
+
         # django-filters has no concept of "abstract" filtersets, so we have to fake it
         if cls._meta.model is not None:
+            if "tags" in filters and isinstance(filters["tags"], TagFilter):
+                filters["tags"].extra["query_params"] = {"content_types": [cls._meta.model._meta.label_lower]}
+
             new_filters = {}
             for existing_filter_name, existing_filter in filters.items():
                 new_filters.update(

nautobot/core/forms/fields.py

@@ -228,6 +228,10 @@ class MultipleContentTypeField(django_forms.ModelMultipleChoiceField):
 
         if "queryset" not in kwargs:
             if feature is not None:
+                from nautobot.extras.registry import registry
+
+                if feature not in registry["model_features"]:
+                    raise KeyError
                 kwargs["queryset"] = ContentType.objects.filter(
                     extras_utils.FeatureQuery(feature).get_query()
                 ).order_by("app_label", "model")
@@ -453,6 +457,7 @@ class DynamicModelChoiceMixin:
 
     filter = django_filters.ModelChoiceFilter  # 2.0 TODO(Glenn): can we rename this? pylint: disable=redefined-builtin
     widget = widgets.APISelect
+    iterator = widgets.MinimalModelChoiceIterator
 
     def __init__(
         self,
@@ -475,6 +480,7 @@ class DynamicModelChoiceMixin:
         # to_field_name is set by ModelChoiceField.__init__(), but we need to set it early for reference
         # by widget_attrs()
         self.to_field_name = kwargs.get("to_field_name")
+        self.data_queryset = kwargs.get("queryset")  # may be updated in get_bound_field()
 
         super().__init__(*args, **kwargs)
 
@@ -539,12 +545,12 @@ class DynamicModelChoiceMixin:
             field_name = getattr(self, "to_field_name") or "pk"
             filter_ = self.filter(field_name=field_name)
             try:
-                self.queryset = filter_.filter(self.queryset, data)
-            except (TypeError, ValidationError):
+                self.data_queryset = filter_.filter(self.queryset, data)
+            except (TypeError, ValueError, ValidationError):
                 # Catch any error caused by invalid initial data passed from the user
-                self.queryset = self.queryset.none()
+                self.data_queryset = self.queryset.none()
         else:
-            self.queryset = self.queryset.none()
+            self.data_queryset = self.queryset.none()
 
         # Set the data URL on the APISelect widget (if not already set)
         widget = bound_field.field.widget

nautobot/core/forms/forms.py

@@ -59,6 +59,13 @@ class AddressFieldMixin(forms.ModelForm):
 class BootstrapMixin(forms.BaseForm):
     """
     Add the base Bootstrap CSS classes to form elements.
+
+    Note that this only applies to form fields that are:
+
+    1. statically defined on the form class at declaration time, or
+    2. dynamically added to the form at init time by a class **later in the MRO than this mixin**.
+
+    If a class earlier in the MRO adds its own fields, it will have to ensure that the widgets are correctly configured.
     """
 
     def __init__(self, *args, **kwargs):
@@ -73,8 +80,9 @@ class BootstrapMixin(forms.BaseForm):
 
         for field in self.fields.values():
             if field.widget.__class__ not in exempt_widgets:
-                css = field.widget.attrs.get("class", "")
-                field.widget.attrs["class"] = " ".join([css, "form-control"]).strip()
+                css_classes = field.widget.attrs.get("class", "")
+                if "form-control" not in css_classes:
+                    field.widget.attrs["class"] = " ".join([css_classes, "form-control"]).strip()
             if field.required and not isinstance(field.widget, forms.FileInput):
                 field.widget.attrs["required"] = "required"
             if "placeholder" not in field.widget.attrs:
@@ -121,7 +129,7 @@ class BulkRenameForm(forms.Form):
     """
 
     find = forms.CharField()
-    replace = forms.CharField()
+    replace = forms.CharField(required=False, strip=False)
     use_regex = forms.BooleanField(required=False, initial=True, label="Use regular expressions")
 
     def clean(self):

nautobot/core/forms/widgets.py

@@ -116,6 +116,23 @@ class ContentTypeSelect(StaticSelect2):
     option_template_name = "widgets/select_contenttype.html"
 
 
+class MinimalModelChoiceIterator(ModelChoiceIterator):
+    """
+    Helper class for APISelect and APISelectMultiple.
+
+    Allows the widget to keep a full `queryset` for data validation, but, for performance reasons, returns a minimal
+    subset of choices at render time derived from the widget's `data_queryset`.
+    """
+
+    @property
+    def queryset(self):
+        return self.field.data_queryset
+
+    @queryset.setter
+    def queryset(self, value):
+        return self.field.data_queryset
+
+
 class APISelect(SelectWithDisabled):
     """
     A select widget populated via an API call
@@ -173,7 +190,7 @@ class APISelect(SelectWithDisabled):
             and isinstance(self.choices, Iterable)
         ):
 
-            class ModelChoiceIteratorWithNullOption(ModelChoiceIterator):
+            class ModelChoiceIteratorWithNullOption(MinimalModelChoiceIterator):
                 def __init__(self, *args, **kwargs):
                     self.null_options = kwargs.pop("null_option", None)
                     super().__init__(*args, **kwargs)

nautobot/core/graphql/schema.py

@@ -4,6 +4,7 @@ from collections import OrderedDict
 import logging
 
 from django.conf import settings
+from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.contenttypes.models import ContentType
 from django.core.validators import ValidationError
 from django.db.models import ManyToManyField
@@ -42,7 +43,7 @@ from nautobot.dcim.graphql.types import (
     RearPortType,
 )
 from nautobot.extras.choices import CustomFieldTypeChoices, RelationshipSideChoices
-from nautobot.extras.graphql.types import DynamicGroupType, TagType
+from nautobot.extras.graphql.types import ContactAssociationType, DynamicGroupType, TagType
 from nautobot.extras.models import ComputedField, CustomField, Relationship
 from nautobot.extras.registry import registry
 from nautobot.extras.utils import check_if_key_is_graphql_safe
@@ -68,8 +69,9 @@ registry["graphql_types"]["dcim.powerport"] = PowerPortType
 registry["graphql_types"]["dcim.rack"] = RackType
 registry["graphql_types"]["dcim.rearport"] = RearPortType
 registry["graphql_types"]["dcim.location"] = LocationType
-registry["graphql_types"]["extras.tag"] = TagType
+registry["graphql_types"]["extras.contactassociation"] = ContactAssociationType
 registry["graphql_types"]["extras.dynamicgroup"] = DynamicGroupType
+registry["graphql_types"]["extras.tag"] = TagType
 registry["graphql_types"]["ipam.ipaddress"] = IPAddressType
 registry["graphql_types"]["ipam.prefix"] = PrefixType
 registry["graphql_types"]["ipam.vlan"] = VLANType
@@ -127,6 +129,11 @@ def extend_schema_type(schema_type):
     #
     schema_type = extend_schema_type_config_context(schema_type, model)
 
+    #
+    # Global features (contacts, teams, dynamic groups)
+    #
+    schema_type = extend_schema_type_global_features(schema_type, model)
+
     #
     # Relationships
     #
@@ -199,8 +206,7 @@ def extend_schema_type_filter(schema_type, model):
         (DjangoObjectType): The extended schema_type object
     """
     for field in model._meta.get_fields():
-        # Check whether attribute is a ManyToOne or ManyToMany field
-        if not isinstance(field, (ManyToManyField, ManyToManyRel, ManyToOneRel)):
+        if not isinstance(field, (ManyToManyField, ManyToManyRel, ManyToOneRel, GenericRelation)):
             continue
         # OneToOneRel is a subclass of ManyToOneRel, but we don't want to treat it as a list
         if isinstance(field, OneToOneRel):
@@ -366,6 +372,22 @@ def extend_schema_type_config_context(schema_type, model):
     return schema_type
 
 
+def extend_schema_type_global_features(schema_type, model):
+    """
+    Extend schema_type object to have attributes and resolvers for global features (dynamic groups, etc.).
+    """
+    # associated_contacts and associated_object_metadata are handled elsewhere by extend_schema_type_filter()
+    if getattr(model, "is_dynamic_group_associable_model", False):
+
+        def resolve_dynamic_groups(self, args):
+            return self.dynamic_groups
+
+        setattr(schema_type, "resolve_dynamic_groups", resolve_dynamic_groups)
+        schema_type._meta.fields["dynamic_groups"] = graphene.Field.mounted(graphene.List(DynamicGroupType))
+
+    return schema_type
+
+
 def extend_schema_type_relationships(schema_type, model):
     """Extend the schema type with attributes and resolvers corresponding
     to the relationships associated with this model."""

nautobot/core/jobs/__init__.py

@@ -15,6 +15,8 @@ from nautobot.core.api.renderers import NautobotCSVRenderer
 from nautobot.core.api.utils import get_serializer_for_model
 from nautobot.core.celery import app, register_jobs
 from nautobot.core.exceptions import AbortTransaction
+from nautobot.core.jobs.cleanup import LogsCleanup
+from nautobot.core.jobs.groups import RefreshDynamicGroupCaches
 from nautobot.core.utils.lookup import get_filterset_for_model
 from nautobot.core.utils.requests import get_filterable_params_from_filter_params
 from nautobot.extras.datasources import ensure_git_repository, git_repository_dry_run, refresh_datasource_content
@@ -37,6 +39,7 @@ class GitRepositorySync(Job):
 
     class Meta:
         name = "Git Repository: Sync"
+        description = "Clone and/or pull a Git repository, then refresh data sourced from this repository."
         has_sensitive_variables = False
 
     def run(self, repository):
@@ -66,6 +69,7 @@ class GitRepositoryDryRun(Job):
 
     class Meta:
         name = "Git Repository: Dry-Run"
+        description = "Dry run of Git repository sync - will not update data sourced from this repository."
         has_sensitive_variables = False
 
     def run(self, repository):
@@ -112,6 +116,7 @@ class ExportObjectList(Job):
 
     class Meta:
         name = "Export Object List"
+        description = "Export a list of objects to CSV or YAML, or render a specified Export Template."
         has_sensitive_variables = False
         # Exporting large querysets may take substantial processing time
         soft_time_limit = 1800
@@ -136,7 +141,15 @@ class ExportObjectList(Job):
         #       such that they never are even seen here.
         query_params = QueryDict(query_string)
         self.logger.debug("Parsed query_params: `%s`", query_params.dict())
-        default_non_filter_params = ("export", "page", "per_page", "sort")
+        default_non_filter_params = (
+            "all_filters_removed",
+            "export",
+            "page",
+            "per_page",
+            "saved_view",
+            "sort",
+            "table_changes_pending",
+        )
         filter_params = get_filterable_params_from_filter_params(
             query_params, default_non_filter_params, filterset_class()
         )
@@ -213,6 +226,7 @@ class ImportObjects(Job):
 
     class Meta:
         name = "Import Objects"
+        description = "Import objects from CSV-formatted data."
         has_sensitive_variables = False
         # Importing large files may take substantial processing time
         soft_time_limit = 1800
@@ -317,5 +331,5 @@ class ImportObjects(Job):
             raise RunJobTaskFailed("CSV import not fully successful, see logs")
 
 
-jobs = [ExportObjectList, GitRepositorySync, GitRepositoryDryRun, ImportObjects]
+jobs = [ExportObjectList, GitRepositorySync, GitRepositoryDryRun, ImportObjects, LogsCleanup, RefreshDynamicGroupCaches]
 register_jobs(*jobs)

nautobot/core/jobs/cleanup.py

@@ -0,0 +1,100 @@
+from datetime import timedelta
+
+from django.core.exceptions import PermissionDenied
+from django.db.models.signals import pre_delete
+from django.utils import timezone
+
+from nautobot.core.choices import ChoiceSet
+from nautobot.core.utils.config import get_settings_or_config
+from nautobot.extras.jobs import IntegerVar, Job, MultiChoiceVar
+from nautobot.extras.models import JobResult, ObjectChange
+from nautobot.extras.signals import _handle_deleted_object
+
+name = "System Jobs"
+
+
+class CleanupTypes(ChoiceSet):
+    JOB_RESULT = "extras.JobResult"
+    OBJECT_CHANGE = "extras.ObjectChange"
+
+    CHOICES = (
+        (JOB_RESULT, "Job results"),
+        (OBJECT_CHANGE, "Change logs"),
+    )
+
+
+class LogsCleanup(Job):
+    """
+    System job to clean up ObjectChange and/or JobResult (and JobLogEntry) records older than a given age.
+    """
+
+    cleanup_types = MultiChoiceVar(
+        choices=CleanupTypes.CHOICES,
+        required=True,
+    )
+
+    max_age = IntegerVar(
+        description=(
+            "Maximum age of records to retain, in days. "
+            "Leave empty to use the CHANGELOG_RETENTION setting as the maximum."
+        ),
+        label="Max Age",
+        min_value=0,
+        required=False,
+    )
+
+    class Meta:
+        name = "Logs Cleanup"
+        description = "Delete ObjectChange and/or JobResult/JobLogEntry records older than a specified cutoff."
+        has_sensitive_variables = False
+
+    def run(self, *, cleanup_types, max_age=None):
+        if max_age in (None, ""):
+            max_age = get_settings_or_config("CHANGELOG_RETENTION")
+            if max_age == 0:
+                self.logger.warning(
+                    "CHANGELOG_RETENTION setting is set to zero, disabling this Job. "
+                    "If you wish to use this Job to delete records, you must specify a `max_age` value."
+                )
+                return 0
+
+        if CleanupTypes.JOB_RESULT in cleanup_types and not self.user.has_perm("extras.delete_jobresult"):
+            self.logger.error('User "%s" does not have permission to delete JobResult records', self.user)
+            raise PermissionDenied("User does not have delete permissions for JobResult records")
+
+        if CleanupTypes.OBJECT_CHANGE in cleanup_types and not self.user.has_perm("extras.delete_objectchange"):
+            self.logger.error('User "%s" does not have permission to delete ObjectChange records', self.user)
+            raise PermissionDenied("User does not have delete permissions for ObjectChange records")
+
+        # Bulk delete goes much faster if Django doesn't have signals to process.
+        # Temporarily detach the ones we *know* to be irrelevant.
+        self.logger.debug("Temporarily disconnecting some signals for performance")
+        pre_delete.disconnect(_handle_deleted_object)
+
+        try:
+            cutoff = timezone.now() - timedelta(days=max_age)
+            result = {}
+
+            if CleanupTypes.JOB_RESULT in cleanup_types:
+                self.logger.info("Deleting JobResult records prior to %s", cutoff)
+                _, deleted_dict = JobResult.objects.restrict(self.user, "delete").filter(date_done__lt=cutoff).delete()
+                result.setdefault("extras.JobResult", 0)
+                result.setdefault("extras.JobLogEntry", 0)
+                result.update(**deleted_dict)
+                self.logger.info(
+                    "Deleted %d JobResult records and their associated %d JobLogEntry records",
+                    result["extras.JobResult"],
+                    result["extras.JobLogEntry"],
+                )
+
+            if CleanupTypes.OBJECT_CHANGE in cleanup_types:
+                self.logger.info("Deleting ObjectChange records prior to %s", cutoff)
+                deleted_count, _ = ObjectChange.objects.restrict(self.user, "delete").filter(time__lt=cutoff).delete()
+                self.logger.info("Deleted %d ObjectChange records", deleted_count)
+                result["extras.ObjectChange"] = deleted_count
+
+            return result
+        finally:
+            # Be sure to clean up after ourselves!
+            self.logger.debug("Re-connecting signals")
+            pre_delete.connect(_handle_deleted_object)

nautobot/core/jobs/groups.py

@@ -0,0 +1,38 @@
+from nautobot.extras.choices import DynamicGroupTypeChoices
+from nautobot.extras.jobs import Job, ObjectVar
+from nautobot.extras.models import DynamicGroup
+
+name = "System Jobs"
+
+
+class RefreshDynamicGroupCaches(Job):
+    """
+    System job to recalculate and re-cache the members of Dynamic Groups for improved performance.
+    """
+
+    single_group = ObjectVar(
+        description="Select to refresh only a single specified group instead of all groups",
+        model=DynamicGroup,
+        query_params={
+            "group_type": [DynamicGroupTypeChoices.TYPE_DYNAMIC_FILTER, DynamicGroupTypeChoices.TYPE_DYNAMIC_SET],
+        },
+        required=False,
+    )
+
+    class Meta:
+        name = "Refresh Dynamic Group Caches"
+        description = "Re-calculate and re-cache the membership lists of Dynamic Groups."
+        has_sensitive_variables = False
+
+    def run(self, single_group=None):
+        groups = DynamicGroup.objects.restrict(self.user, "view").exclude(
+            group_type=DynamicGroupTypeChoices.TYPE_STATIC
+        )
+        if single_group is not None:
+            groups = groups.filter(pk=single_group.pk)
+
+        for group in groups:
+            group.update_cached_members()
+            self.logger.info("Cache refreshed successfully, now with %d members", group.count, extra={"object": group})
+
+        self.logger.info("Cache(s) refreshed")