nautobot 2.2.2__py3-none-any.whl → 2.2.3__py3-none-any.whl

nautobot/extras/jobs.py

@@ -6,18 +6,14 @@ import inspect
 import json
 import logging
 import os
+import sys
 import tempfile
 from textwrap import dedent
 from typing import final
 import warnings
 
-from billiard.einfo import ExceptionInfo, ExceptionWithTraceback
-from celery import states
-from celery.exceptions import NotRegistered, Retry
-from celery.result import EagerResult
-from celery.utils.functional import maybe_list
+from billiard.einfo import ExceptionInfo
 from celery.utils.log import get_task_logger
-from celery.utils.nodenames import gethostname
 from db_file_storage.form_widgets import DBClearableFileInput
 from django import forms
 from django.conf import settings
@@ -30,12 +26,10 @@ from django.db.models import Model
 from django.db.models.query import QuerySet
 from django.forms import ValidationError
 from django.utils.functional import classproperty
-from kombu.utils.uuid import uuid
 import netaddr
 import yaml
 
-from nautobot.core.celery import app as celery_app
-from nautobot.core.celery.task import Task
+from nautobot.core.celery import import_jobs, nautobot_task
 from nautobot.core.forms import (
     DynamicModelChoiceField,
     DynamicModelMultipleChoiceField,
@@ -53,6 +47,7 @@ from nautobot.extras.models import (
     JobResult,
     ObjectChange,
 )
+from nautobot.extras.registry import registry
 from nautobot.extras.utils import change_logged_models_queryset, task_queues_as_choices
 from nautobot.ipam.formfields import IPAddressFormField, IPNetworkFormField
 from nautobot.ipam.validators import (
@@ -88,7 +83,7 @@ class RunJobTaskFailed(Exception):
     """Celery task failed for some reason."""
 
 
-class BaseJob(Task):
+class BaseJob:
     """Base model for jobs.
 
     Users can subclass this directly if they want to provide their own base class for implementing multiple jobs
@@ -158,38 +153,6 @@ class BaseJob(Task):
 
     # See https://github.com/PyCQA/pylint-django/issues/240 for why we have a pylint disable on each classproperty below
 
-    # TODO(jathan): Could be interesting for custom stuff when the Job is
-    # enabled in the database and then therefore registered in Celery
-    @classmethod
-    def on_bound(cls, app):
-        """Called when the task is bound to an app.
-
-        Note:
-            This class method can be defined to do additional actions when
-            the task class is bound to an app.
-        """
-
-    # TODO(jathan): Could be interesting for showing the Job's class path as the
-    # shadow name vs. the Celery task_name?
-    def shadow_name(self, args, kwargs, options):
-        """Override for custom task name in worker logs/monitoring.
-
-        Example:
-                from celery.utils.imports import qualname
-
-                def shadow_name(task, args, kwargs, options):
-                    return qualname(args[0])
-
-                @app.task(shadow_name=shadow_name, serializer='pickle')
-                def apply_function_async(fun, *args, **kwargs):
-                    return fun(*args, **kwargs)
-
-        Arguments:
-            args (Tuple): Task positional arguments.
-            kwargs (Dict): Task keyword arguments.
-            options (Dict): Task execution options.
-        """
-
     def before_start(self, task_id, args, kwargs):
         """Handler called before the task starts.
 
@@ -201,8 +164,6 @@ class BaseJob(Task):
         Returns:
             (None): The return value of this handler is ignored.
         """
-        self.clear_cache()
-
         try:
             self.job_result
         except ObjectDoesNotExist as err:
@@ -234,7 +195,7 @@ class BaseJob(Task):
                 extra={"grouping": "initialization"},
             )
 
-        self.logger.info("Running job", extra={"grouping": "initialization"})
+        self.logger.info("Running job", extra={"grouping": "initialization", "object": self.job_model})
 
     def run(self, *args, **kwargs):
         """
@@ -314,84 +275,10 @@ class BaseJob(Task):
         if status == JobResultStatusChoices.STATUS_SUCCESS:
             self.logger.info("Job completed", extra={"grouping": "post_run"})
 
-        # TODO(gary): document this in job author docs
-        # Super.after_return must be called for chords to function properly
-        super().after_return(status, retval, task_id, args, kwargs, einfo=einfo)
-
-    def apply(
-        self,
-        args=None,
-        kwargs=None,
-        link=None,
-        link_error=None,
-        task_id=None,
-        retries=None,
-        throw=None,
-        logfile=None,
-        loglevel=None,
-        headers=None,
-        **options,
-    ):
-        """Fix celery's apply method to propagate options to the task result"""
-        # trace imports Task, so need to import inline.
-        from celery.app.trace import build_tracer
-
-        app = self._get_app()
-        args = args or ()
-        kwargs = kwargs or {}
-        task_id = task_id or uuid()
-        retries = retries or 0
-        if throw is None:
-            throw = app.conf.task_eager_propagates
-
-        # Make sure we get the task instance, not class.
-        task = app._tasks[self.name]
-
-        request = {
-            "id": task_id,
-            "retries": retries,
-            "is_eager": True,
-            "logfile": logfile,
-            "loglevel": loglevel or 0,
-            "hostname": gethostname(),
-            "callbacks": maybe_list(link),
-            "errbacks": maybe_list(link_error),
-            "headers": headers,
-            "ignore_result": options.get("ignore_result", False),
-            "delivery_info": {
-                "is_eager": True,
-                "exchange": options.get("exchange"),
-                "routing_key": options.get("routing_key"),
-                "priority": options.get("priority"),
-            },
-            "properties": options,  # one line fix to overloaded method
-        }
-        if "stamped_headers" in options:
-            request["stamped_headers"] = maybe_list(options["stamped_headers"])
-            request["stamps"] = {header: maybe_list(options.get(header, [])) for header in request["stamped_headers"]}
-
-        tb = None
-        tracer = build_tracer(
-            task.name,
-            task,
-            eager=True,
-            propagate=throw,
-            app=self._get_app(),
-        )
-        ret = tracer(task_id, args, kwargs, request)
-        retval = ret.retval
-        if isinstance(retval, ExceptionInfo):
-            retval, tb = retval.exception, retval.traceback
-            if isinstance(retval, ExceptionWithTraceback):
-                retval = retval.exc
-        if isinstance(retval, Retry) and retval.sig is not None:
-            return retval.sig.apply(retries=retries + 1)
-        state = states.SUCCESS if ret.info is None else ret.info.state
-        return EagerResult(task_id, retval, state, traceback=tb)
-
     @final
     @classproperty
     def file_path(cls) -> str:  # pylint: disable=no-self-argument
+        """Deprecated as of Nautobot 2.2.3."""
         return inspect.getfile(cls)
 
     @final
@@ -430,7 +317,7 @@ class BaseJob(Task):
     @classproperty
     def grouping(cls) -> str:  # pylint: disable=no-self-argument
         module = inspect.getmodule(cls)
-        return getattr(module, "name", module.__name__)
+        return getattr(module, "name", cls.__module__)
 
     @final
     @classmethod
@@ -530,6 +417,7 @@ class BaseJob(Task):
     @final
     @classproperty
     def registered_name(cls) -> str:  # pylint: disable=no-self-argument
+        """Deprecated - use class_path classproperty instead."""
         return f"{cls.__module__}.{cls.__name__}"
 
     @classmethod
@@ -545,7 +433,10 @@ class BaseJob(Task):
         base_classes = reversed(inspect.getmro(cls))
         attr_names = [name for base in base_classes for name in base.__dict__.keys()]
         for name in attr_names:
-            attr_class = getattr(cls, name, None).__class__
+            try:
+                attr_class = getattr(cls, name, None).__class__
+            except TypeError:
+                pass
             if name not in cls_vars and issubclass(attr_class, ScriptVariable):
                 cls_vars[name] = getattr(cls, name)
 
@@ -612,27 +503,9 @@ class BaseJob(Task):
 
         return form
 
-    def clear_cache(self):
-        """
-        Clear all cached properties on this instance without accessing them. This is required because
-        celery reuses task instances for multiple runs.
-        """
-        try:
-            del self.celery_kwargs
-        except AttributeError:
-            pass
-        try:
-            del self.job_result
-        except AttributeError:
-            pass
-        try:
-            del self.job_model
-        except AttributeError:
-            pass
-
     @functools.cached_property
     def job_model(self):
-        return JobModel.objects.get(module_name=self.__module__, job_class_name=self.__name__)
+        return JobModel.objects.get(module_name=self.__module__, job_class_name=self.__class__.__name__)
 
     @functools.cached_property
     def job_result(self):
@@ -1199,16 +1072,76 @@ def is_variable(obj):
     return isinstance(obj, ScriptVariable)
 
 
-def get_job(class_path):
+def get_jobs(*, reload=False):
+    """
+    Compile a dictionary of all Job classes available at this time.
+
+    Args:
+        reload (bool): If True, reimport Jobs from `JOBS_ROOT` and all applicable GitRepositories.
+
+    Returns:
+        (dict): `{"class_path.Job1": <job_class>, "class_path.Job2": <job_class>, ...}`
+    """
+    if reload:
+        import_jobs()
+
+    return registry["jobs"]
+
+
+def get_job(class_path, reload=False):
     """
     Retrieve a specific job class by its class_path (`<module_name>.<JobClassName>`).
 
-    May return None if the job isn't properly registered with Celery at this time.
+    May return None if the job can't be imported.
+
+    Args:
+        reload (bool): If True, **and** the given class_path describes a JOBS_ROOT or GitRepository Job,
+            then refresh **all** such Jobs before retrieving the job class.
+    """
+    if reload:
+        if class_path.startswith("nautobot."):
+            # System job - not reloadable
+            reload = False
+        if any(class_path.startswith(f"{app_name}.") for app_name in settings.PLUGINS):
+            # App provided job - not reloadable
+            reload = False
+    jobs = get_jobs(reload=reload)
+    return jobs.get(class_path, None)
+
+
+@nautobot_task(bind=True)
+def run_job(self, job_class_path, *args, **kwargs):
+    """
+    "Runner" function for execution of any Job class by a worker.
+
+    This calls the following Job APIs in the following order:
+
+    - `__init__()`
+    - `before_start()`
+    - `__call__()` (which calls `run()`)
+    - If no exceptions have been raised, `on_success()`, else `on_failure()`
+    - `after_return()`
+
+    Finally, it either returns the data returned from `run()` or re-raises any exception encountered.
     """
+    logger.debug("Running job %s", job_class_path)
+
+    job_class = get_job(job_class_path, reload=True)
+    if job_class is None:
+        raise KeyError(f"Job class not found for class path {job_class_path}")
+    job = job_class()
+    job.request = self.request
     try:
-        return celery_app.tasks[class_path].__class__
-    except NotRegistered:
-        return None
+        job.before_start(self.request.id, args, kwargs)
+        result = job(*args, **kwargs)
+        job.on_success(result, self.request.id, args, kwargs)
+        job.after_return(JobResultStatusChoices.STATUS_SUCCESS, result, self.request.id, args, kwargs, None)
+        return result
+    except Exception as exc:
+        einfo = ExceptionInfo(sys.exc_info())
+        job.on_failure(exc, self.request.id, args, kwargs, einfo)
+        job.after_return(JobResultStatusChoices.STATUS_FAILURE, exc, self.request.id, args, kwargs, einfo)
+        raise
 
 
 def enqueue_job_hooks(object_change):

nautobot/extras/models/datasources.py

@@ -97,8 +97,6 @@ class GitRepository(PrimaryModel):
         if not self.present_in_database:
             check_if_key_is_graphql_safe(self.__class__.__name__, self.slug, "slug")
             # Check on create whether the proposed slug conflicts with a module name already in the Python environment.
-            # Because we add GIT_ROOT to the end of sys.path, trying to import this repository will instead
-            # import the earlier-found Python module in its place, which would be undesirable.
             if find_spec(self.slug) is not None:
                 raise ValidationError(
                     f'Please choose a different slug, as "{self.slug}" is an installed Python package or module.'

nautobot/extras/models/jobs.py

@@ -5,6 +5,7 @@ from datetime import timedelta
 import logging
 
 from celery import schedules
+from celery.exceptions import NotRegistered
 from celery.utils.log import get_logger, LoggingProxy
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
@@ -22,7 +23,6 @@ from nautobot.core.celery import (
     NautobotKombuJSONEncoder,
     setup_nautobot_job_logging,
 )
-from nautobot.core.celery.control import refresh_git_repository
 from nautobot.core.constants import CHARFIELD_MAX_LENGTH
 from nautobot.core.models import BaseManager, BaseModel
 from nautobot.core.models.fields import JSONArrayField
@@ -233,13 +233,20 @@ class Job(PrimaryModel):
     def __str__(self):
         return self.name
 
-    @cached_property
+    @property
     def job_class(self):
-        """Get the Job class (source code) associated with this Job model."""
+        """
+        Get the Job class (source code) associated with this Job model.
+
+        CAUTION: if the Job is provided by a Git Repository or is installed in JOBS_ROOT, you may need or wish to
+        call `get_job(self.class_path, reload=True)` to ensure that you have the latest Job code...
+        """
+        from nautobot.extras.jobs import get_job
+
         if not self.installed:
             return None
         try:
-            return self.job_task.__class__
+            return get_job(self.class_path)
         except Exception as exc:
             logger.error(str(exc))
             return None
@@ -278,20 +285,23 @@ class Job(PrimaryModel):
 
     @property
     def job_task(self):
-        """Get the registered Celery task, refreshing it if necessary."""
-        if self.git_repository is not None:
-            # If this Job comes from a Git repository, make sure we have the correct version of said code.
-            refresh_git_repository(
-                state=None, repository_pk=self.git_repository.pk, head=self.git_repository.current_head
-            )
-        return app.tasks[f"{self.module_name}.{self.job_class_name}"]
+        """Get an instance of the associated Job class, refreshing it if necessary."""
+        from nautobot.extras.jobs import get_job
+
+        try:
+            return get_job(self.class_path, reload=True)()
+        except TypeError as err:  # keep 2.0-2.2.2 exception behavior
+            raise NotRegistered from err
 
     def clean(self):
         """For any non-overridden fields, make sure they get reset to the actual underlying class value if known."""
-        if self.job_class is not None:
+        from nautobot.extras.jobs import get_job
+
+        job_class = get_job(self.class_path, reload=True)
+        if job_class is not None:
             for field_name in JOB_OVERRIDABLE_FIELDS:
                 if not getattr(self, f"{field_name}_override", False):
-                    setattr(self, field_name, getattr(self.job_class, field_name))
+                    setattr(self, field_name, getattr(job_class, field_name))
 
         # Protect against invalid input when auto-creating Job records
         if len(self.module_name) > JOB_MAX_NAME_LENGTH:
@@ -617,12 +627,14 @@ class JobResult(BaseModel, CustomFieldModel):
             schedule (ScheduledJob, optional): ScheduledJob instance to link to the JobResult. Cannot be used with synchronous=True.
             task_queue (str, optional): The celery queue to send the job to. If not set, use the default celery queue.
             synchronous (bool, optional): If True, run the job in the current process, blocking until the job completes.
-            *job_args: positional args passed to the job task
+            *job_args: positional args passed to the job task (UNUSED)
             **job_kwargs: keyword args passed to the job task
 
         Returns:
             JobResult instance
         """
+        from nautobot.extras.jobs import run_job  # TODO circular import
+
         if schedule is not None and synchronous:
             raise ValueError("Scheduled jobs cannot be run synchronously")
 
@@ -666,8 +678,11 @@ class JobResult(BaseModel, CustomFieldModel):
             redirect_logger = get_logger("celery.redirected")
             proxy = LoggingProxy(redirect_logger, app.conf.worker_redirect_stdouts_level)
             with contextlib.redirect_stdout(proxy), contextlib.redirect_stderr(proxy):
-                eager_result = job_model.job_task.apply(
-                    args=job_args, kwargs=job_kwargs, task_id=str(job_result.id), **job_celery_kwargs
+                eager_result = run_job.apply(
+                    args=[job_model.class_path, *job_args],
+                    kwargs=job_kwargs,
+                    task_id=str(job_result.id),
+                    **job_celery_kwargs,
                 )
 
             # copy fields from eager result to job result
@@ -687,8 +702,11 @@ class JobResult(BaseModel, CustomFieldModel):
         else:
             # Jobs queued inside of a transaction need to run after the transaction completes and the JobResult is saved to the database
             transaction.on_commit(
-                lambda: job_model.job_task.apply_async(
-                    args=job_args, kwargs=job_kwargs, task_id=str(job_result.id), **job_celery_kwargs
+                lambda: run_job.apply_async(
+                    args=[job_model.class_path, *job_args],
+                    kwargs=job_kwargs,
+                    task_id=str(job_result.id),
+                    **job_celery_kwargs,
                 )
             )
 

nautobot/extras/plugins/__init__.py

@@ -31,7 +31,6 @@ logger = getLogger(__name__)
 registry["plugin_banners"] = []
 registry["plugin_custom_validators"] = collections.defaultdict(list)
 registry["plugin_graphql_types"] = []
-registry["plugin_jobs"] = []
 registry["plugin_template_extensions"] = collections.defaultdict(list)
 registry["app_metrics"] = []
 
@@ -141,9 +140,9 @@ class NautobotAppConfig(NautobotConfig):
             register_graphql_types(graphql_types)
 
         # Import jobs (if present)
+        # Note that we do *not* auto-call `register_jobs()` - the App is responsible for doing so when imported.
         jobs = import_object(f"{self.__module__}.{self.jobs}")
         if jobs is not None:
-            register_jobs(jobs)
             self.features["jobs"] = jobs
 
         # Import metrics (if present)
@@ -423,24 +422,6 @@ def register_graphql_types(class_list):
         registry["plugin_graphql_types"].append(item)
 
 
-def register_jobs(class_list):
-    """
-    Register a list of Job classes
-    """
-    from nautobot.extras.jobs import Job
-
-    for job in class_list:
-        if not inspect.isclass(job):
-            raise TypeError(f"Job class {job} was passed as an instance!")
-        if not issubclass(job, Job):
-            raise TypeError(f"{job} is not a subclass of extras.jobs.Job!")
-
-        registry["plugin_jobs"].append(job)
-
-    # Note that we do not (and cannot) update the Job records in the Nautobot database at this time.
-    # That is done in response to the `nautobot_database_ready` signal, see nautobot.extras.signals.refresh_job_models
-
-
 def register_metrics(function_list):
     """
     Register a list of metric functions

nautobot/extras/signals.py

@@ -21,7 +21,7 @@ from django.utils import timezone
 from django_prometheus.models import model_deletes, model_inserts, model_updates
 import redis.exceptions
 
-from nautobot.core.celery import app, import_jobs_as_celery_tasks
+from nautobot.core.celery import app, import_jobs
 from nautobot.core.models import BaseModel
 from nautobot.core.utils.config import get_settings_or_config
 from nautobot.core.utils.logging import sanitize
@@ -333,7 +333,7 @@ def git_repository_pre_delete(instance, **kwargs):
         app.control.broadcast("discard_git_repository", repository_slug=instance.slug)
         # But we don't have an equivalent way to broadcast to any other Django instances.
         # For now we just delete the one that we have locally and rely on other methods,
-        # such as the import_jobs_as_celery_tasks() signal that runs on server startup,
+        # such as the import_jobs() signal that runs on server startup,
         # to clean up other clones as they're encountered.
         if os.path.isdir(instance.filesystem_path):
             shutil.rmtree(instance.filesystem_path)
@@ -462,7 +462,7 @@ def refresh_job_models(sender, *, apps, **kwargs):
     """
     Callback for the nautobot_database_ready signal; updates Jobs in the database based on Job source file availability.
     """
-    from nautobot.extras.jobs import Job as JobClass  # avoid circular import
+    from nautobot.extras.jobs import get_jobs  # avoid circular import
 
     Job = apps.get_model("extras", "Job")
 
@@ -471,15 +471,12 @@ def refresh_job_models(sender, *, apps, **kwargs):
         logger.info("Skipping refresh_job_models() as it appears Job model has not yet been migrated to latest.")
         return
 
-    import_jobs_as_celery_tasks(app)
+    import_jobs()
 
     job_models = []
-    for task in app.tasks.values():
-        # Skip Celery tasks that aren't Jobs
-        if not isinstance(task, JobClass):
-            continue
 
-        job_model, _ = refresh_job_model_from_job_class(Job, task.__class__)
+    for job_class in get_jobs().values():
+        job_model, _ = refresh_job_model_from_job_class(Job, job_class)
         if job_model is not None:
             job_models.append(job_model)
 

nautobot/extras/test_jobs/__init__.py

@@ -0,0 +1,8 @@
+def load_tests(*args):
+    """Implement unittest discovery for this submodule as a no-op.
+
+    This prevents unittest from recursively loading all of the modules under this directory to inspect whether they
+    define test cases. This is necessary because otherwise the `jobs_module` submodule will get loaded when tests run,
+    which will in turn call `register_jobs()`, incorrectly/unexpectedly registering the test Job defined in that module
+    as if it were a system Job, which will cause tests to fail due to the unexpected presence of this Job.
+    """

nautobot/extras/test_jobs/dry_run.py

@@ -1,5 +1,5 @@
 from nautobot.core.celery import register_jobs
-from nautobot.extras.jobs import DryRunVar, get_task_logger, Job
+from nautobot.extras.jobs import DryRunVar, get_task_logger, IntegerVar, Job
 from nautobot.extras.models import Status
 
 logger = get_task_logger(__name__)
@@ -11,8 +11,9 @@ class TestDryRun(Job):
     """
 
     dryrun = DryRunVar()
+    value = IntegerVar(required=False)
 
-    def run(self, dryrun):
+    def run(self, dryrun, value=None):
         """
         Job function.
         """

nautobot/extras/test_jobs/fail.py

@@ -1,4 +1,7 @@
+from billiard.einfo import ExceptionInfo
+
 from nautobot.core.celery import register_jobs
+from nautobot.extras.choices import JobResultStatusChoices
 from nautobot.extras.jobs import get_task_logger, Job, RunJobTaskFailed
 
 logger = get_task_logger(__name__)
@@ -11,6 +14,15 @@ class TestFail(Job):
 
     description = "Validate job import"
 
+    def before_start(self, task_id, args, kwargs):
+        if task_id != self.request.id:
+            raise RuntimeError(f"Expected task_id {task_id} to equal self.request.id {self.request.id}")
+        if args:
+            raise RuntimeError(f"Expected args to be empty, but it was {args!r}")
+        if kwargs:
+            raise RuntimeError(f"Expected kwargs to be empty, but it was {kwargs!r}")
+        logger.info("before_start() was called as expected")
+
     def run(self):
         """
         Job function.
@@ -18,6 +30,37 @@ class TestFail(Job):
         logger.info("I'm a test job that fails!")
         raise RunJobTaskFailed("Test failure")
 
+    def on_success(self, retval, task_id, args, kwargs):
+        raise RuntimeError("on_success() was unexpectedly called!")
+
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        if not isinstance(exc, RunJobTaskFailed):
+            raise RuntimeError(f"Expected exc to be a RunJobTaskFailed, but it was {exc!r}")
+        if task_id != self.request.id:
+            raise RuntimeError(f"Expected task_id {task_id} to equal self.request.id {self.request.id}")
+        if args:
+            raise RuntimeError(f"Expected args to be empty, but it was {args!r}")
+        if kwargs:
+            raise RuntimeError(f"Expected kwargs to be empty, but it was {kwargs!r}")
+        if not isinstance(einfo, ExceptionInfo):
+            raise RuntimeError(f"Expected einfo to be an ExceptionInfo, but it was {einfo!r}")
+        logger.info("on_failure() was called as expected")
+
+    def after_return(self, status, retval, task_id, args, kwargs, einfo):
+        if status is not JobResultStatusChoices.STATUS_FAILURE:
+            raise RuntimeError(f"Expected status to be {JobResultStatusChoices.STATUS_FAILURE}, but it was {status!r}")
+        if not isinstance(retval, RunJobTaskFailed):
+            raise RuntimeError(f"Expected retval to be a RunJobTaskFailed, but it was {retval!r}")
+        if task_id != self.request.id:
+            raise RuntimeError(f"Expected task_id {task_id} to equal self.request.id {self.request.id}")
+        if args:
+            raise RuntimeError(f"Expected args to be empty, but it was {args!r}")
+        if kwargs:
+            raise RuntimeError(f"Expected kwargs to be empty, but it was {kwargs!r}")
+        if not isinstance(einfo, ExceptionInfo):
+            raise RuntimeError(f"Expected einfo to be an ExceptionInfo, but it was {einfo!r}")
+        logger.info("after_return() was called as expected")
+
 
 class TestFailWithSanitization(Job):
     """