nautobot 2.3.16__py3-none-any.whl → 2.4.0b1__py3-none-any.whl

nautobot/extras/context_managers.py

@@ -6,6 +6,7 @@ from django.contrib.auth.models import AnonymousUser
 from django.db import transaction
 from django.test.client import RequestFactory
 
+from nautobot.core.events import publish_event
 from nautobot.extras.choices import ObjectChangeEventContextChoices
 from nautobot.extras.constants import CHANGELOG_MAX_CHANGE_CONTEXT_DETAIL
 from nautobot.extras.models import ObjectChange
@@ -211,27 +212,43 @@ def web_request_context(
         last_action = None
         last_content_type = None
         # enqueue jobhooks and webhooks, use change_context.change_id in case change_id was not supplied
-        for object_change in (
+        for oc in (
             ObjectChange.objects.select_related("changed_object_type", "user")
             .filter(request_id=change_context.change_id)
             .order_by("time")  # default ordering is -time but we want oldest first not newest first
             .iterator()
         ):
-            if object_change.action != last_action or object_change.changed_object_type != last_content_type:
+            if oc.action != last_action or oc.changed_object_type != last_content_type:
                 jobhook_queryset = None
                 webhook_queryset = None
 
             if context != ObjectChangeEventContextChoices.CONTEXT_JOB_HOOK:
                 # Make sure JobHooks are up to date (only once) before calling them
                 did_reload_jobs, jobhook_queryset = enqueue_job_hooks(
-                    object_change, may_reload_jobs=(not jobs_reloaded), jobhook_queryset=jobhook_queryset
+                    oc, may_reload_jobs=(not jobs_reloaded), jobhook_queryset=jobhook_queryset
                 )
                 if did_reload_jobs:
                     jobs_reloaded = True
 
-            webhook_queryset = enqueue_webhooks(object_change, webhook_queryset=webhook_queryset)
-            last_action = object_change.action
-            last_content_type = object_change.changed_object_type
+            # TODO: get_snapshots() currently requires a DB query per object change processed.
+            # We need to develop a more efficient approach: https://github.com/nautobot/nautobot/issues/6303
+            snapshots = oc.get_snapshots()
+            webhook_queryset = enqueue_webhooks(oc, snapshots=snapshots, webhook_queryset=webhook_queryset)
+
+            # topic examples: "nautobot.change.dcim.device", "nautobot.add.ipam.ipaddress"
+            event_topic = f"nautobot.{oc.action}.{oc.changed_object_type.app_label}.{oc.changed_object_type.model}"
+            event_payload = snapshots.copy()
+            event_payload["context"] = {
+                "change_context": oc.get_change_context_display(),
+                "change_context_detail": oc.change_context_detail,
+                "request_id": str(oc.request_id),
+                "user_name": oc.user_name,
+                "timestamp": str(oc.time),
+            }
+            publish_event(topic=event_topic, payload=event_payload)
+
+            last_action = oc.action
+            last_content_type = oc.changed_object_type
 
 
 @contextmanager

nautobot/extras/datasources/git.py

@@ -31,6 +31,7 @@ from nautobot.extras.models import (
     ExportTemplate,
     GitRepository,
     Job,
+    JobQueue,
     JobResult,
     Role,
     Tag,
@@ -763,7 +764,9 @@ def refresh_git_jobs(repository_record, job_result, delete=False):
                 if not job_class_path.startswith(f"{repository_record.slug}.jobs."):
                     continue
                 found_jobs = True
-                job_model, created = refresh_job_model_from_job_class(Job, job_class)
+                job_model, created = refresh_job_model_from_job_class(
+                    job_model_class=Job, job_class=job_class, job_queue_class=JobQueue
+                )
 
                 if job_model is None:
                     msg = "Failed to create Job record; check Nautobot logs for details"

nautobot/extras/factory.py

@@ -20,6 +20,7 @@ from nautobot.core.factory import (
 from nautobot.core.templatetags.helpers import bettertitle
 from nautobot.extras.choices import (
     DynamicGroupTypeChoices,
+    JobQueueTypeChoices,
     JobResultStatusChoices,
     LogLevelChoices,
     MetadataTypeDataTypeChoices,
@@ -34,6 +35,7 @@ from nautobot.extras.models import (
     ExternalIntegration,
     Job,
     JobLogEntry,
+    JobQueue,
     JobResult,
     MetadataChoice,
     MetadataType,
@@ -140,6 +142,31 @@ class JobLogEntryFactory(BaseModelFactory):
         return faker.Faker().past_datetime(start_date=self.job_result.date_created, tzinfo=timezone.utc)
 
 
+class JobQueueFactory(PrimaryModelFactory):
+    """JobQueue model Factory"""
+
+    class Meta:
+        model = JobQueue
+
+    class Params:
+        has_description = NautobotBoolIterator()
+        has_tenant = NautobotBoolIterator()
+
+    name = factory.LazyAttributeSequence(lambda o, n: f"{o.queue_type} Job Queue - {n}")
+    description = factory.Maybe("has_description", factory.Faker("text", max_nb_chars=CHARFIELD_MAX_LENGTH), "")
+    queue_type = factory.Iterator(JobQueueTypeChoices.CHOICES, getter=lambda choice: choice[0])
+    tenant = factory.Maybe("has_tenant", random_instance(Tenant))
+
+    @factory.post_generation
+    def jobs(self, create, extracted, **kwargs):
+        jobs = get_random_instances(Job)
+        self.jobs.set(jobs)
+        for job in jobs:
+            if not job.job_queues_override:
+                job.job_queues_override = True
+                job.validated_save()
+
+
 class JobResultFactory(BaseModelFactory):
     """JobResult model factory."""
 

nautobot/extras/filters/__init__.py

@@ -23,6 +23,7 @@ from nautobot.core.filters import (
 from nautobot.core.utils.deprecation import class_deprecated_in_favor_of
 from nautobot.dcim.models import DeviceRedundancyGroup, DeviceType, Location, Platform
 from nautobot.extras.choices import (
+    JobQueueTypeChoices,
     JobResultStatusChoices,
     MetadataTypeDataTypeChoices,
     RelationshipTypeChoices,
@@ -73,6 +74,8 @@ from nautobot.extras.models import (
     JobButton,
     JobHook,
     JobLogEntry,
+    JobQueue,
+    JobQueueAssignment,
     JobResult,
     MetadataChoice,
     MetadataType,
@@ -133,6 +136,8 @@ __all__ = (
     "ImageAttachmentFilterSet",
     "JobFilterSet",
     "JobLogEntryFilterSet",
+    "JobQueueFilterSet",
+    "JobQueueAssignmentFilterSet",
     "JobResultFilterSet",
     "LocalContextFilterSet",
     "LocalContextModelFilterSetMixin",
@@ -906,6 +911,60 @@ class JobHookFilterSet(BaseFilterSet):
         ]
 
 
+class JobQueueFilterSet(NautobotFilterSet, TenancyModelFilterSetMixin):
+    q = SearchFilter(
+        filter_predicates={
+            "name": "icontains",
+            "queue_type": "icontains",
+            "description": "icontains",
+            "tenant__name": "icontains",
+        },
+    )
+    queue_type = django_filters.MultipleChoiceFilter(choices=JobQueueTypeChoices, null_value=None)
+    jobs = NaturalKeyOrPKMultipleChoiceFilter(
+        queryset=Job.objects.all(),
+        label="Job (name or ID)",
+    )
+    has_jobs = RelatedMembershipBooleanFilter(
+        field_name="jobs",
+        label="Has jobs",
+    )
+
+    class Meta:
+        model = JobQueue
+        fields = [
+            "id",
+            "name",
+            "description",
+            "tags",
+        ]
+
+
+class JobQueueAssignmentFilterSet(BaseFilterSet):
+    q = SearchFilter(
+        filter_predicates={
+            "job__name": "icontains",
+            "job__grouping": "icontains",
+            "job__description": "icontains",
+            "job_queue__name": "icontains",
+            "job_queue__description": "icontains",
+            "job_queue__queue_type": "icontains",
+        }
+    )
+    job = NaturalKeyOrPKMultipleChoiceFilter(
+        queryset=Job.objects.all(),
+        label="Job (name or ID)",
+    )
+    job_queue = NaturalKeyOrPKMultipleChoiceFilter(
+        queryset=JobQueue.objects.all(),
+        label="Job Queue (name or ID)",
+    )
+
+    class Meta:
+        model = JobQueueAssignment
+        fields = ["id"]
+
+
 class JobResultFilterSet(BaseFilterSet, CustomFieldModelFilterSetMixin):
     q = SearchFilter(
         filter_predicates={

nautobot/extras/forms/forms.py

@@ -9,6 +9,7 @@ from django.core.validators import MinValueValidator
 from django.db.models.fields import TextField
 from django.forms import inlineformset_factory, ModelMultipleChoiceField
 from django.urls.base import reverse
+from django.utils.timezone import get_current_timezone_name
 
 from nautobot.core.constants import CHARFIELD_MAX_LENGTH
 from nautobot.core.forms import (
@@ -25,7 +26,6 @@ from nautobot.core.forms import (
     DateTimePicker,
     DynamicModelChoiceField,
     DynamicModelMultipleChoiceField,
-    JSONArrayFormField,
     JSONField,
     LaxURLField,
     MultipleContentTypeField,
@@ -40,6 +40,7 @@ from nautobot.dcim.models import Device, DeviceRedundancyGroup, DeviceType, Loca
 from nautobot.extras.choices import (
     DynamicGroupTypeChoices,
     JobExecutionType,
+    JobQueueTypeChoices,
     JobResultStatusChoices,
     ObjectChangeActionChoices,
     RelationshipTypeChoices,
@@ -65,6 +66,7 @@ from nautobot.extras.models import (
     Job,
     JobButton,
     JobHook,
+    JobQueue,
     JobResult,
     MetadataChoice,
     MetadataType,
@@ -106,6 +108,7 @@ from .mixins import (
     CustomFieldModelFormMixin,
     NoteModelBulkEditFormMixin,
     NoteModelFormMixin,
+    TagsBulkEditFormMixin,
 )
 
 __all__ = (
@@ -149,6 +152,9 @@ __all__ = (
     "JobFilterForm",
     "JobHookForm",
     "JobHookFilterForm",
+    "JobQueueBulkEditForm",
+    "JobQueueFilterForm",
+    "JobQueueForm",
     "JobScheduleForm",
     "JobResultFilterForm",
     "LocalContextFilterForm",
@@ -945,28 +951,19 @@ class JobForm(BootstrapMixin, forms.Form):
     controlled by the job definition. See `nautobot.extras.jobs.BaseJob.as_form`
     """
 
-    _profile = forms.BooleanField(
-        required=False,
-        initial=False,
-        label="Profile job execution",
-        help_text="Profiles the job execution using cProfile and outputs a report to /tmp/",
+
+class JobEditForm(NautobotModelForm):
+    job_queues = DynamicModelMultipleChoiceField(
+        label="Job Queues",
+        queryset=JobQueue.objects.all(),
     )
-    _task_queue = forms.ChoiceField(
+    default_job_queue = DynamicModelChoiceField(
+        label="Default Job Queue",
+        queryset=JobQueue.objects.all(),
+        help_text="The default job queue to route this job to",
         required=False,
-        help_text="The task queue to route this job to",
-        label="Task queue",
     )
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        # Move special fields to the end of the form
-        for field in ["_task_queue", "_profile"]:
-            value = self.fields.pop(field)
-            self.fields[field] = value
-
-
-class JobEditForm(NautobotModelForm):
     class Meta:
         model = Job
         fields = [
@@ -989,8 +986,10 @@ class JobEditForm(NautobotModelForm):
             "time_limit",
             "has_sensitive_variables_override",
             "has_sensitive_variables",
-            "task_queues_override",
-            "task_queues",
+            "job_queues_override",
+            "job_queues",
+            "default_job_queue_override",
+            "default_job_queue",
             "tags",
         ]
 
@@ -1006,8 +1005,28 @@ class JobEditForm(NautobotModelForm):
             for field_name in JOB_OVERRIDABLE_FIELDS:
                 if not cleaned_data.get(f"{field_name}_override", False):
                     cleaned_data[field_name] = getattr(job_class, field_name)
+            # Get default Job Queue first
+            if not cleaned_data.get("default_job_queue_override", False):
+                meta_task_queues = getattr(job_class, "task_queues", []) or [settings.CELERY_TASK_DEFAULT_QUEUE]
+                cleaned_data["default_job_queue"], _ = JobQueue.objects.get_or_create(
+                    name=meta_task_queues[0], defaults={"queue_type": JobQueueTypeChoices.TYPE_CELERY}
+                )
+            default_job_queue = cleaned_data["default_job_queue"]
+            # Include the default Job Queue in the Job Queues selection
+            if not cleaned_data.get("job_queues_override", False):
+                names = getattr(job_class, "task_queues", []) or [settings.CELERY_TASK_DEFAULT_QUEUE]
+            else:
+                names = list(cleaned_data["job_queues"].values_list("name", flat=True))
+            names += [default_job_queue]
+            cleaned_data["job_queues"] = JobQueue.objects.filter(name__in=names)
+
         return cleaned_data
 
+    def save(self, *args, **kwargs):
+        instance = super().save(*args, **kwargs)
+        instance.job_queues.set(self.cleaned_data["job_queues"])
+        return instance
+
 
 class JobBulkEditForm(NautobotBulkEditForm):
     """Bulk edit form for `Job` objects."""
@@ -1058,10 +1077,17 @@ class JobBulkEditForm(NautobotBulkEditForm):
         help_text="Maximum runtime in seconds before the job will be forcibly terminated."
         "<br>Set to 0 to use Nautobot system default",
     )
-    task_queues = JSONArrayFormField(
-        base_field=forms.CharField(max_length=CHARFIELD_MAX_LENGTH),
-        help_text="Comma separated list of task queues that this job can run on. A blank list will use the default queue",
+    job_queues = DynamicModelMultipleChoiceField(
+        label="Job Queues",
+        queryset=JobQueue.objects.all(),
+        required=False,
+        help_text="Job Queue instances that this job can run on",
+    )
+    default_job_queue = DynamicModelChoiceField(
+        label="Default Job Queue",
+        queryset=JobQueue.objects.all(),
         required=False,
+        help_text="Default Job Queue the job runs on if no Job Queue is specified",
     )
     # Flags to indicate whether the above properties are inherited from the source code or overridden by the database
     # Text field overrides
@@ -1081,9 +1107,13 @@ class JobBulkEditForm(NautobotBulkEditForm):
         required=False,
         help_text="If checked, time limits will be reverted to the default values defined in each Job's source code",
     )
-    clear_task_queues_override = forms.BooleanField(
+    clear_job_queues_override = forms.BooleanField(
         required=False,
-        help_text="If checked, task queue overrides will be reverted to the default values defined in each Job's source code",
+        help_text="If checked, the selected job queues will be reverted to the default values defined in each Job's source code",
+    )
+    clear_default_job_queue_override = forms.BooleanField(
+        required=False,
+        help_text="If checked, the default job queue will be reverted to the first value of task_queues defined in each Job's source code",
     )
     # Boolean overrides
     clear_approval_required_override = forms.BooleanField(
@@ -1201,6 +1231,64 @@ class JobHookFilterForm(BootstrapMixin, forms.Form):
     type_delete = forms.NullBooleanField(required=False, widget=StaticSelect2(choices=BOOLEAN_WITH_BLANK_CHOICES))
 
 
+class JobQueueBulkEditForm(TagsBulkEditFormMixin, NautobotBulkEditForm):
+    pk = forms.ModelMultipleChoiceField(
+        queryset=JobQueue.objects.all(),
+        widget=forms.MultipleHiddenInput(),
+    )
+    queue_type = forms.ChoiceField(
+        choices=JobQueueTypeChoices,
+        help_text="The job can either run immediately, once in the future, or on a recurring schedule.",
+        label="Type",
+        required=False,
+    )
+    tenant = DynamicModelChoiceField(
+        queryset=Tenant.objects.all(),
+        required=False,
+    )
+    description = forms.CharField(required=False, max_length=CHARFIELD_MAX_LENGTH)
+
+    class Meta:
+        model = JobQueue
+        nullable_fields = [
+            "secrets_group",
+            "description",
+            "tenant",
+        ]
+
+
+class JobQueueFilterForm(NautobotFilterForm):
+    model = JobQueue
+    q = forms.CharField(required=False, label="Search")
+    name = forms.CharField(required=False)
+    jobs = DynamicModelMultipleChoiceField(queryset=Job.objects.all(), required=False)
+    queue_type = forms.MultipleChoiceField(
+        choices=JobQueueTypeChoices,
+        required=False,
+        widget=StaticSelect2Multiple(),
+    )
+    tenant = DynamicModelMultipleChoiceField(queryset=Tenant.objects.all(), to_field_name="name", required=False)
+    tags = TagFilterField(model)
+
+
+class JobQueueForm(NautobotModelForm):
+    name = forms.CharField(required=True, max_length=CHARFIELD_MAX_LENGTH)
+    queue_type = forms.ChoiceField(
+        choices=JobQueueTypeChoices,
+        label="Queue Type",
+    )
+    tenant = DynamicModelChoiceField(
+        queryset=Tenant.objects.all(),
+        required=False,
+    )
+    description = forms.CharField(required=False, max_length=CHARFIELD_MAX_LENGTH)
+    tags = DynamicModelMultipleChoiceField(queryset=Tag.objects.all(), required=False)
+
+    class Meta:
+        model = JobQueue
+        fields = ("name", "queue_type", "description", "tenant", "tags")
+
+
 class JobScheduleForm(BootstrapMixin, forms.Form):
     """
     This form is rendered alongside the JobForm but deals specifically with the fields needed to either
@@ -1223,7 +1311,6 @@ class JobScheduleForm(BootstrapMixin, forms.Form):
         required=False,
         label="Starting date and time",
         widget=DateTimePicker(),
-        help_text=f"The scheduled time is relative to the Nautobot configured timezone: {settings.TIME_ZONE}.",
     )
     _recurrence_custom_time = forms.CharField(
         required=False,
@@ -1260,6 +1347,16 @@ class JobScheduleForm(BootstrapMixin, forms.Form):
                 except Exception as e:
                     raise ValidationError({"_recurrence_custom_time": e})
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # setting the help_text for `_schedule_start_time` here instead of in the field definition
+        # because Django needs to be fully initialized before we can accurately retrieve the current timezone.
+        self.fields[
+            "_schedule_start_time"
+        ].help_text = (
+            f"The scheduled time is relative to the Nautobot configured timezone: {get_current_timezone_name()}."
+        )
+
 
 class JobResultFilterForm(BootstrapMixin, forms.Form):
     model = JobResult
@@ -1429,7 +1526,7 @@ class MetadataTypeFilterForm(NautobotFilterForm):
     tags = TagFilterField(model)
 
 
-class MetadataTypeBulkEditForm(NautobotBulkEditForm):
+class MetadataTypeBulkEditForm(TagsBulkEditFormMixin, NautobotBulkEditForm):
     pk = forms.ModelMultipleChoiceField(queryset=MetadataType.objects.all(), widget=forms.MultipleHiddenInput)
     description = forms.CharField(required=False, max_length=CHARFIELD_MAX_LENGTH)
 
@@ -1682,8 +1779,6 @@ class RoleBulkEditForm(NautobotBulkEditForm):
 
     pk = forms.ModelMultipleChoiceField(queryset=Role.objects.all(), widget=forms.MultipleHiddenInput)
     color = forms.CharField(max_length=6, required=False, widget=ColorSelect())
-    description = forms.CharField(max_length=CHARFIELD_MAX_LENGTH, required=False)
-    weight = forms.IntegerField(required=False)
     content_types = MultipleContentTypeField(
         queryset=RoleModelsQuery().as_queryset(), required=False, label="Content Type(s)"
     )

nautobot/extras/forms/mixins.py

@@ -837,21 +837,13 @@ class StatusModelFilterFormMixin(forms.Form):
         self.order_fields(self.field_order)  # Reorder fields again
 
 
-class TagsBulkEditFormMixin(BulkEditForm):
+class TagsBulkEditFormMixin(forms.Form):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         # Add add/remove tags fields
-        self.fields["add_tags"] = DynamicModelMultipleChoiceField(
-            queryset=Tag.objects.all(),
-            query_params={"content_types": self.model._meta.label_lower},
-            required=False,
-        )
-        self.fields["remove_tags"] = DynamicModelMultipleChoiceField(
-            queryset=Tag.objects.all(),
-            query_params={"content_types": self.model._meta.label_lower},
-            required=False,
-        )
+        self.fields["add_tags"] = DynamicModelMultipleChoiceField(queryset=Tag.objects.all(), required=False)
+        self.fields["remove_tags"] = DynamicModelMultipleChoiceField(queryset=Tag.objects.all(), required=False)
 
 
 # 2.2 TODO: Names below are only for backward compatibility with Nautobot 1.3 and earlier. Remove in 2.2

nautobot/extras/graphql/types.py

@@ -1,11 +1,15 @@
+import graphene
+
 from nautobot.core.graphql.types import OptimizedNautobotObjectType
 from nautobot.extras.filters import (
     ContactAssociationFilterSet,
     DynamicGroupFilterSet,
+    JobFilterSet,
+    ScheduledJobFilterSet,
     StatusFilterSet,
     TagFilterSet,
 )
-from nautobot.extras.models import ContactAssociation, DynamicGroup, Status, Tag
+from nautobot.extras.models import ContactAssociation, DynamicGroup, Job, ScheduledJob, Status, Tag
 
 
 class ContactAssociationType(OptimizedNautobotObjectType):
@@ -24,6 +28,26 @@ class DynamicGroupType(OptimizedNautobotObjectType):
         filterset_class = DynamicGroupFilterSet
 
 
+class JobType(OptimizedNautobotObjectType):
+    """Graphql Type Object for Job model."""
+
+    class Meta:
+        model = Job
+        filterset_class = JobFilterSet
+
+    task_queues = graphene.List(graphene.String)
+
+
+class ScheduledJobType(OptimizedNautobotObjectType):
+    """Graphql Type Object for Job model."""
+
+    class Meta:
+        model = ScheduledJob
+        filterset_class = ScheduledJobFilterSet
+
+    queue = graphene.String
+
+
 class StatusType(OptimizedNautobotObjectType):
     """Graphql Type object for `Status` model."""
 

nautobot/extras/group_sync.py

@@ -8,15 +8,15 @@ from django.contrib.auth.models import Group
 logger = logging.getLogger(__name__)
 
 
-CLAIMS_GROUP_NAME = getattr(settings, "SSO_CLAIMS_GROUP", "groups")
+CLAIMS_GROUP_NAME = getattr(settings, "NAUTOBOT_SSO_CLAIMS_GROUP", "groups")
 """ Which claim to look at in the OAuth2/OIDC response
 
     For Okta you can look at `Okta -> Authorization Servers -> Claims`. And a reasonable
     default is "groups". For Azure a reasonable default is "roles".
 """
 
-SUPERUSER_GROUPS = getattr(settings, "SSO_SUPERUSER_GROUPS", [])
-STAFF_GROUPS = getattr(settings, "SSO_STAFF_GROUPS", [])
+SUPERUSER_GROUPS = getattr(settings, "NAUTOBOT_SSO_SUPERUSER_GROUPS", [])
+STAFF_GROUPS = getattr(settings, "NAUTOBOT_SSO_STAFF_GROUPS", [])
 
 
 def group_sync(uid, user=None, response=None, *args, **kwargs):

nautobot/extras/health_checks.py

@@ -160,4 +160,5 @@ class RedisBackend(RedisHealthCheck):
         elif client_class == "django_redis.client.DefaultClient":
             self.check_redis(redis_url=location, **options.get("CONNECTION_POOL_KWARGS", {}))
         else:
-            self.add_error(ServiceUnavailable(f"{client_class} is an unsupported CLIENT_CLASS!"))
+            if self.redis_url is None:
+                self.add_error(ServiceUnavailable(f"{client_class} is an unsupported CLIENT_CLASS!"))