nautobot 2.3.15b1__py3-none-any.whl → 2.4.0__py3-none-any.whl

nautobot/core/celery/schedulers.py

@@ -1,14 +1,20 @@
 from collections.abc import Mapping
 from datetime import datetime, timedelta
+import json
 import logging
 from pathlib import Path
+import sys
 
 from celery import current_app
+from celery.beat import _evaluate_entry_args, _evaluate_entry_kwargs, reraise, SchedulingError
+from celery.result import AsyncResult
 from django.conf import settings
 from django_celery_beat.schedulers import DatabaseScheduler, ModelEntry
 from kombu.utils.json import loads
 
-from nautobot.extras.models import ScheduledJob, ScheduledJobs
+from nautobot.extras.choices import JobQueueTypeChoices
+from nautobot.extras.models import JobResult, ScheduledJob, ScheduledJobs
+from nautobot.extras.utils import run_kubernetes_job_and_return_job_result
 
 logger = logging.getLogger(__name__)
 
@@ -19,7 +25,7 @@ class NautobotScheduleEntry(ModelEntry):
     nautobot.extras.models.ScheduledJob model
     """
 
-    def __init__(self, model, app=None):
+    def __init__(self, model, app=None):  # pylint:disable=super-init-not-called  # we must copy-and-paste from super
         """Initialize the model entry."""
         # copy-paste from django_celery_beat.schedulers
         self.app = app or current_app._get_current_object()
@@ -111,7 +117,46 @@ class NautobotDatabaseScheduler(DatabaseScheduler):
 
         Ref: https://github.com/celery/django-celery-beat/issues/558#issuecomment-1162730008
         """
-        resp = super().apply_async(entry, producer=producer, advance=advance, **kwargs)
+        resp = None
+        entry = self.reserve(entry) if advance else entry
+        task = self.app.tasks.get(entry.task)
+
+        try:
+            entry_args = _evaluate_entry_args(entry.args)
+            entry_kwargs = _evaluate_entry_kwargs(entry.kwargs)
+            if task:
+                scheduled_job = entry.model
+                job_queue = scheduled_job.job_queue
+                # Distinguish between Celery and Kubernetes job queues
+                if job_queue.queue_type == JobQueueTypeChoices.TYPE_KUBERNETES:
+                    job_result = JobResult.objects.create(
+                        name=scheduled_job.job_model.name,
+                        job_model=scheduled_job.job_model,
+                        scheduled_job=scheduled_job,
+                        user=scheduled_job.user,
+                        task_name=scheduled_job.job_model.class_path,
+                        celery_kwargs=entry.options,
+                    )
+                    job_result = run_kubernetes_job_and_return_job_result(
+                        job_queue, job_result, json.dumps(entry_kwargs)
+                    )
+                    # Return an AsyncResult object to mimic the behavior of Celery tasks after the job is finished by Kubernetes Job Pod.
+                    resp = AsyncResult(job_result.id)
+                else:
+                    resp = task.apply_async(entry_args, entry_kwargs, producer=producer, **entry.options)
+            else:
+                resp = self.send_task(entry.task, entry_args, entry_kwargs, producer=producer, **entry.options)
+        except Exception as exc:  # pylint: disable=broad-except
+            reraise(
+                SchedulingError,
+                SchedulingError(f"Couldn't apply scheduled task {entry.name}: {exc}"),
+                sys.exc_info()[2],
+            )
+        finally:
+            self._tasks_since_sync += 1
+            if self.should_sync():
+                self._do_sync()
+
         if entry.total_run_count != entry.model.total_run_count:
             entry.total_run_count = entry.model.total_run_count
             entry.model.save()

nautobot/core/choices.py

@@ -182,14 +182,14 @@ class ButtonActionColorChoices(ChoiceSet):
     Map standard button actions to Bootstrap color classes.
     """
 
-    ADD = "info"
+    ADD = "primary"
     CANCEL = "default"
     CLONE = "success"
     CONFIGURE = "default"
     CONNECT = "success"
     DEFAULT = "default"
     DELETE = "danger"
-    DISCONNECT = "info"
+    DISCONNECT = "danger"
     EDIT = "warning"
     EXPORT = "success"
     IMPORT = "primary"

nautobot/core/cli/__init__.py

@@ -13,6 +13,7 @@ from django.core.management import CommandError, CommandParser, execute_from_com
 from django.core.management.utils import get_random_secret_key
 from jinja2 import BaseLoader, Environment
 
+from nautobot.core.events import load_event_brokers
 from nautobot.core.settings_funcs import is_truthy
 from nautobot.extras.plugins.utils import load_plugins
 
@@ -44,6 +45,7 @@ def _preprocess_settings(settings, config_path):
     - Set up 'job_logs' database mirror
     - Handle our custom `STORAGE_BACKEND` setting.
     - Load plugins based on settings.PLUGINS (potentially affecting INSTALLED_APPS, MIDDLEWARE, and CONSTANCE_CONFIG)
+    - Load event brokers based on settings.EVENT_BROKERS
     """
     settings.SETTINGS_PATH = config_path
 
@@ -124,6 +126,12 @@ def _preprocess_settings(settings, config_path):
     # passed in.
     load_plugins(settings)
 
+    #
+    # Event Broker
+    #
+
+    load_event_brokers(settings.EVENT_BROKERS)
+
 
 def load_settings(config_path):
     """Load nautobot_config.py or its equivalent into memory as a `nautobot_config` pseudo-module."""

nautobot/core/constants.py

@@ -110,6 +110,10 @@ CONFIG_SETTING_SEPARATOR = ","
 
 CHARFIELD_MAX_LENGTH = 255
 
+# Default values for pagination settings.
+MAX_PAGE_SIZE_DEFAULT = 1000
+PAGINATE_COUNT_DEFAULT = 50
+
 # Models excluded from the global search list
 GLOBAL_SEARCH_EXCLUDE_LIST = [
     "anotherexamplemodel",
@@ -146,12 +150,15 @@ GLOBAL_SEARCH_EXCLUDE_LIST = [
     "interfaceredundancygroup",
     "interfaceredundancygroupassociation",
     "interfacetemplate",
+    "interfacevdcassignment",
     "inventoryitem",
     "ipaddresstointerface",
     "job",
     "jobbutton",
     "jobhook",
     "joblogentry",
+    "jobqueue",
+    "jobqueueassignment",
     "jobresult",
     "locationtype",
     "manufacturer",

nautobot/core/events/__init__.py

@@ -0,0 +1,116 @@
+"""Module providing for the publication of event notifications via mechanisms such as Redis, Kafka, syslog, etc."""
+
+import fnmatch
+import json
+import logging
+
+from django.utils.module_loading import import_string
+
+from nautobot.core.celery import NautobotKombuJSONEncoder
+from nautobot.core.events.exceptions import EventBrokerImproperlyConfigured, EventBrokerNotFound
+
+from .base import EventBroker
+from .redis_broker import RedisEventBroker
+from .syslog_broker import SyslogEventBroker
+
+_EVENT_BROKERS = []
+
+
+logger = logging.getLogger(__name__)
+
+
+def load_event_brokers(event_broker_configs):
+    """Process plugins and log errors if they can't be loaded."""
+    for broker_name, broker in event_broker_configs.items():
+        options = broker.get("OPTIONS", {})
+        topics = broker.get("TOPICS", {})
+        if not isinstance(topics, dict):
+            raise EventBrokerImproperlyConfigured(
+                f"{broker_name} Malformed Event Broker Settings: Expected `TOPICS` to be a 'dict', instead a '{type(topics).__name__}' was provided"
+            )
+        include_topics = topics.get("INCLUDE")
+        if include_topics and not isinstance(include_topics, (list, tuple)):
+            raise EventBrokerImproperlyConfigured(
+                f"{broker_name} Malformed Event Broker Settings: Expected `INCLUDE` to be a 'list' or 'tuple', instead a '{type(include_topics).__name__}' was provided"
+            )
+        exclude_topics = topics.get("EXCLUDE", [])
+        if exclude_topics and not isinstance(exclude_topics, (list, tuple)):
+            raise EventBrokerImproperlyConfigured(
+                f"{broker_name} Malformed Event Broker Settings: Expected `EXCLUDE` to be a 'list' or 'tuple', instead a '{type(exclude_topics).__name__}' was provided"
+            )
+        options.update({"include_topics": include_topics, "exclude_topics": exclude_topics})
+
+        try:
+            event_broker_class = import_string(broker["CLASS"])
+            if not issubclass(event_broker_class, EventBroker):
+                raise EventBrokerImproperlyConfigured(
+                    f"{broker_name} Malformed Event Broker Settings: Broker provided is not an EventBroker"
+                )
+            event_broker = event_broker_class(**options)
+            register_event_broker(event_broker)
+        except ImportError as err:
+            raise EventBrokerNotFound(f"Unable to import Event Broker {broker_name}.") from err
+
+
+def register_event_broker(event_broker):
+    """
+    Register an `EventBroker` instance for use by Nautobot.
+
+    The `publish_event()` API will publish events to each registered broker.
+    The expectation/intent here, at least initially, is that a given deployment will instantiate zero or more
+    EventBrokers, then call `register_event_broker()` for each one, as part of Nautobot initial startup.
+
+    Args:
+        event_broker (EventBroker): The initialized/configured EventBroker instance to register.
+    """
+    if event_broker not in _EVENT_BROKERS:
+        _EVENT_BROKERS.append(event_broker)
+        logger.debug("Registered %s as an event broker", event_broker)
+    else:
+        logger.warning("Tried to register event broker %s but it was already registered", event_broker)
+
+
+def deregister_event_broker(event_broker):
+    """
+    Deregister a previously registered `EventBroker` instance so that it no longer receives published events.
+    """
+    try:
+        _EVENT_BROKERS.remove(event_broker)
+        logger.debug("Deregistered event broker %s", event_broker)
+    except ValueError:
+        logger.warning("Tried to deregister event broker %s but it wasn't previously registered", event_broker)
+
+
+def is_topic_match(topic, patterns):
+    return any(fnmatch.fnmatch(topic, pattern) for pattern in patterns)
+
+
+def publish_event(*, topic, payload):
+    """Publish the given event payload to the given topic via all registered `EventBroker` instances.
+
+    Args:
+        topic (str): Topic identifier.
+            Convention is to use snake_case and use periods to delineate related groups of topics,
+            similar to Python logger naming. For example, you might use `nautobot.create.dcim.device`,
+            `nautobot.update.dcim.device`, `nautobot.delete.dcim.device`, `nautobot.create.dcim.interface`,
+            `nautobot.create.ipam.ipaddress`, etc.
+        payload (dict): JSON-serializable structured data to publish.
+            While not all EventBrokers may actually use JSON as their data format, it makes for a reasonable
+            lowest common denominator for serializability.
+    """
+    for event_broker in _EVENT_BROKERS:
+        exclude_topics = event_broker.exclude_topics
+        include_topics = event_broker.include_topics
+        if is_topic_match(topic, include_topics) and not is_topic_match(topic, exclude_topics):
+            serialized_payload = json.dumps(payload, cls=NautobotKombuJSONEncoder)
+            event_broker.publish(topic=topic, payload=serialized_payload)
+
+
+__all__ = (
+    "EventBroker",
+    "RedisEventBroker",
+    "SyslogEventBroker",
+    "deregister_event_broker",
+    "publish_event",
+    "register_event_broker",
+)

nautobot/core/events/base.py

@@ -0,0 +1,27 @@
+"""Base classes for Nautobot event notification framework."""
+
+from abc import ABC, abstractmethod
+
+
+class EventBroker(ABC):
+    """Abstract base class for concrete implementations of event brokers such as syslog, Redis, Kafka, etc."""
+
+    def __init__(self, *args, include_topics=None, exclude_topics=None, **kwargs) -> None:
+        self.include_topics = include_topics or ["*"]
+        self.exclude_topics = exclude_topics or []
+        super().__init__(*args, **kwargs)
+
+    @abstractmethod
+    def publish(self, *, topic, payload):
+        """
+        Possibly publish the given data `payload` to the given event `topic`.
+
+        Args:
+            topic (str): Topic identifier.
+                Convention is to use snake_case and use periods to delineate related groups of topics,
+                similar to Python logger naming. For example, you might receive topics such as `create.dcim.device`,
+                `update.dcim.device`, `delete.dcim.device`, `create.dcim.interface`, `create.ipam.ipaddress`, etc.
+            payload (dict): JSON-serializable structured data to publish.
+                While not all EventBrokers may actually use JSON as their data format, it makes for a reasonable
+                lowest common denominator for serializability.
+        """

nautobot/core/events/exceptions.py

@@ -0,0 +1,10 @@
+class EventBrokerError(Exception):
+    """Base exception for all event-broker-related errors."""
+
+
+class EventBrokerNotFound(EventBrokerError):
+    """Raised when a specified event broker module cannot be found."""
+
+
+class EventBrokerImproperlyConfigured(EventBrokerError):
+    """Raised when a event is not properly configured."""

nautobot/core/events/redis_broker.py

@@ -0,0 +1,48 @@
+"""Simple EventBroker for publishing events to Redis.
+
+To verify that an instance of this broker is working, you can do the following in `nautobot-server nbshell`:
+
+```python
+import json
+
+import redis
+
+
+connection = redis.StrictRedis(host="redis", port=6379, db=2, password="...", charset="utf-8", decode_responses=True)
+
+
+def listen():
+    sub = connection.pubsub()
+    sub.psubscribe("nautobot.*")
+    for message in sub.listen():
+        if message["type"] == "pmessage":
+            print(f"Got a pmessage on channel {message['channel']}")
+            print(json.dumps(json.loads(message["data"]), indent=2))
+
+
+listen()
+```
+
+Then perform any action that triggers an event, such as creating or updating a record through the Nautobot UI.
+"""
+
+import redis
+
+from .base import EventBroker
+
+
+class RedisEventBroker(EventBroker):
+    """EventBroker for publishing events to Redis."""
+
+    def __init__(self, *args, url, **kwargs):
+        """Initialize and configure a RedisEventBroker.
+
+        Args:
+            url (str): The Redis URL to connect to.
+        """
+        self.url = url
+        self.connection = redis.StrictRedis.from_url(self.url)
+        super().__init__(*args, **kwargs)
+
+    def publish(self, *, topic, payload):
+        self.connection.publish(topic, payload)

nautobot/core/events/syslog_broker.py

@@ -0,0 +1,19 @@
+"""Simple EventBroker that just emits syslogs."""
+
+import json
+import logging
+
+from .base import EventBroker
+
+
+class SyslogEventBroker(EventBroker):
+    def __init__(self, *args, level=logging.INFO, **kwargs):
+        """Initialize a SyslogEventBroker that emits logs at the given level."""
+        super().__init__(*args, **kwargs)
+        self.level = level
+
+    def publish(self, *, topic, payload):
+        logger = logging.getLogger(f"nautobot.events.{topic}")
+        # Add an indent of 4 to payload
+        payload = json.dumps(json.loads(payload), indent=4)
+        logger.log(self.level, "%s", payload)

nautobot/core/exceptions.py

@@ -33,9 +33,3 @@ class FilterSetFieldNotFound(Exception):
     """
     An exception indicating that a filterset field could not be found.
     """
-
-
-class ViewConfigException(Exception):
-    """
-    An exception indicating that a detail view config is invalid.
-    """

nautobot/core/filters.py

@@ -668,7 +668,7 @@ class BaseFilterSet(django_filters.FilterSet):
         the form `<field_name>__<lookup_expr>`
         """
         magic_filters = {}
-        if filter_field.method is not None or filter_field.lookup_expr not in ["exact", "in"]:
+        if filter_field.method is not None or filter_field.lookup_expr not in ["exact", "in", "iexact"]:
             return magic_filters
 
         # Choose the lookup expression map based on the filter type
@@ -679,7 +679,7 @@ class BaseFilterSet(django_filters.FilterSet):
 
         # Get properties of the existing filter for later use
         field_name = filter_field.field_name
-        field = get_model_field(cls._meta.model, field_name)
+        field = get_model_field(cls._meta.model, field_name)  # pylint: disable=no-member
 
         # If there isn't a model field, return.
         if field is None:
@@ -696,7 +696,7 @@ class BaseFilterSet(django_filters.FilterSet):
             new_filter_name = f"{filter_name}__{lookup_name}"
 
             try:
-                if filter_name in cls.declared_filters and lookup_expr not in {"isnull"}:
+                if filter_name in cls.declared_filters and lookup_expr not in {"isnull"}:  # pylint: disable=no-member
                     # The filter field has been explicitly defined on the filterset class so we must manually
                     # create the new filter with the same type because there is no guarantee the defined type
                     # is the same as the default type for the field. This does not apply if the filter
@@ -727,7 +727,10 @@ class BaseFilterSet(django_filters.FilterSet):
             # If the base filter_field has a custom label, django_filters won't adjust it for the new_filter lookup,
             # so we have to do it.
             if filter_field.label and filter_field.label != label_for_filter(
-                cls.Meta.model, filter_field.field_name, filter_field.lookup_expr, filter_field.exclude
+                cls._meta.model,  # pylint: disable=no-member
+                filter_field.field_name,
+                filter_field.lookup_expr,
+                filter_field.exclude,
             ):
                 # Lightly adjusted from label_for_filter() implementation:
                 verbose_expression = ["exclude", filter_field.label] if new_filter.exclude else [filter_field.label]
@@ -750,22 +753,22 @@ class BaseFilterSet(django_filters.FilterSet):
         if not isinstance(new_filter_field, django_filters.Filter):
             raise TypeError(f"Tried to add filter ({new_filter_name}) which is not an instance of Django Filter")
 
-        if new_filter_name in cls.base_filters:
+        if new_filter_name in cls.base_filters:  # pylint: disable=no-member
             raise AttributeError(
                 f"There was a conflict with filter `{new_filter_name}`, the custom filter was ignored."
             )
 
-        cls.base_filters[new_filter_name] = new_filter_field
+        cls.base_filters[new_filter_name] = new_filter_field  # pylint: disable=no-member
         # django-filters has no concept of "abstract" filtersets, so we have to fake it
-        if cls._meta.model is not None:
-            cls.base_filters.update(
+        if cls._meta.model is not None:  # pylint: disable=no-member
+            cls.base_filters.update(  # pylint: disable=no-member
                 cls._generate_lookup_expression_filters(filter_name=new_filter_name, filter_field=new_filter_field)
             )
 
     @classmethod
     def get_fields(cls):
         fields = super().get_fields()
-        if "id" not in fields and (cls._meta.exclude is None or "id" not in cls._meta.exclude):
+        if "id" not in fields and (cls._meta.exclude is None or "id" not in cls._meta.exclude):  # pylint: disable=no-member
             # Add "id" as the first key in the `fields` dict
             fields = {"id": [django_filters.conf.settings.DEFAULT_LOOKUP_EXPR], **fields}
         return fields
@@ -782,7 +785,7 @@ class BaseFilterSet(django_filters.FilterSet):
             if filter_name.startswith("_"):
                 del filters[filter_name]
 
-        if getattr(cls._meta.model, "is_contact_associable_model", False):
+        if getattr(cls._meta.model, "is_contact_associable_model", False):  # pylint: disable=no-member
             # Add "contacts" and "teams" filters
             from nautobot.extras.models import Contact, Team
 
@@ -802,13 +805,13 @@ class BaseFilterSet(django_filters.FilterSet):
                     label="Teams (name or ID)",
                 )
 
-        if "dynamic_groups" not in filters and getattr(cls._meta.model, "is_dynamic_group_associable_model", False):
-            if not hasattr(cls._meta.model, "static_group_association_set"):
+        if "dynamic_groups" not in filters and getattr(cls._meta.model, "is_dynamic_group_associable_model", False):  # pylint: disable=no-member
+            if not hasattr(cls._meta.model, "static_group_association_set"):  # pylint: disable=no-member
                 logger.warning(
                     "Model %s has 'is_dynamic_group_associable_model = True' but lacks "
                     "a 'static_group_association_set' attribute. Perhaps this is due to it inheriting from "
                     "the deprecated DynamicGroupMixin class instead of the preferred DynamicGroupsModelMixin?",
-                    cls._meta.model,
+                    cls._meta.model,  # pylint: disable=no-member
                 )
             else:
                 # Add "dynamic_groups" field as the last key
@@ -818,14 +821,14 @@ class BaseFilterSet(django_filters.FilterSet):
                     queryset=DynamicGroup.objects.all(),
                     field_name="static_group_association_set__dynamic_group",
                     to_field_name="name",
-                    query_params={"content_type": cls._meta.model._meta.label_lower},
+                    query_params={"content_type": cls._meta.model._meta.label_lower},  # pylint: disable=no-member
                     label="Dynamic groups (name or ID)",
                 )
 
         # django-filters has no concept of "abstract" filtersets, so we have to fake it
-        if cls._meta.model is not None:
+        if cls._meta.model is not None:  # pylint: disable=no-member
             if "tags" in filters and isinstance(filters["tags"], TagFilter):
-                filters["tags"].extra["query_params"] = {"content_types": [cls._meta.model._meta.label_lower]}
+                filters["tags"].extra["query_params"] = {"content_types": [cls._meta.model._meta.label_lower]}  # pylint: disable=no-member
 
             new_filters = {}
             for existing_filter_name, existing_filter in filters.items():

nautobot/core/forms/__init__.py

@@ -76,24 +76,21 @@ from nautobot.core.forms.widgets import (
 )
 
 __all__ = (
-    "add_blank_choice",
-    "add_field_to_filter_form_class",
-    "AddressFieldMixin",
     "ALPHANUMERIC_EXPANSION_PATTERN",
+    "BOOLEAN_CHOICES",
+    "BOOLEAN_WITH_BLANK_CHOICES",
+    "IP4_EXPANSION_PATTERN",
+    "IP6_EXPANSION_PATTERN",
+    "NUMERIC_EXPANSION_PATTERN",
     "APISelect",
     "APISelectMultiple",
+    "AddressFieldMixin",
     "AutoPositionField",
     "AutoPositionPatternField",
-    "BOOLEAN_CHOICES",
-    "BOOLEAN_WITH_BLANK_CHOICES",
     "BootstrapMixin",
     "BulkEditForm",
     "BulkEditNullBooleanSelect",
     "BulkRenameForm",
-    "ColorSelect",
-    "CommentField",
-    "ConfirmationForm",
-    "ContentTypeSelect",
     "CSVChoiceField",
     "CSVContentTypeField",
     "CSVDataField",
@@ -102,34 +99,29 @@ __all__ = (
     "CSVModelForm",
     "CSVMultipleChoiceField",
     "CSVMultipleContentTypeField",
+    "ColorSelect",
+    "CommentField",
+    "ConfirmationForm",
+    "ContentTypeSelect",
     "DatePicker",
     "DateTimePicker",
     "DynamicFilterForm",
     "DynamicModelChoiceField",
     "DynamicModelMultipleChoiceField",
-    "expand_alphanumeric_pattern",
-    "expand_ipaddress_pattern",
     "ExpandableIPAddressField",
     "ExpandableNameField",
-    "form_from_model",
     "ImportForm",
-    "IP4_EXPANSION_PATTERN",
-    "IP6_EXPANSION_PATTERN",
     "JSONArrayFormField",
     "JSONField",
     "LaxURLField",
     "MACAddressField",
     "MultiMatchModelMultipleChoiceField",
-    "MultipleContentTypeField",
     "MultiValueCharField",
     "MultiValueCharInput",
+    "MultipleContentTypeField",
     "NullableDateField",
-    "NUMERIC_EXPANSION_PATTERN",
     "NumericArrayField",
-    "parse_alphanumeric_range",
-    "parse_numeric_range",
     "PrefixFieldMixin",
-    "restrict_form_fields",
     "ReturnURLForm",
     "SearchForm",
     "SelectWithDisabled",
@@ -142,4 +134,12 @@ __all__ = (
     "TableConfigForm",
     "TagFilterField",
     "TimePicker",
+    "add_blank_choice",
+    "add_field_to_filter_form_class",
+    "expand_alphanumeric_pattern",
+    "expand_ipaddress_pattern",
+    "form_from_model",
+    "parse_alphanumeric_range",
+    "parse_numeric_range",
+    "restrict_form_fields",
 )