nautobot 2.0.0a3__py3-none-any.whl → 2.0.0b1__py3-none-any.whl

nautobot/extras/datasources/git.py

@@ -5,6 +5,7 @@ import logging
 import mimetypes
 import os
 import re
+import sys
 from urllib.parse import quote
 
 from django.conf import settings
@@ -14,13 +15,10 @@ from django.db import transaction
 from django.utils.text import slugify
 import yaml
 
-from nautobot.core.celery import nautobot_task
+from nautobot.core.celery import app as celery_app
 from nautobot.core.utils.git import GitRepo
-from nautobot.core.utils.requests import copy_safe_request
 from nautobot.dcim.models import Device, DeviceType, Location, Platform
 from nautobot.extras.choices import (
-    JobSourceChoices,
-    JobResultStatusChoices,
     LogLevelChoices,
     SecretsGroupAccessTypeChoices,
     SecretsGroupSecretTypeChoices,
@@ -32,16 +30,14 @@ from nautobot.extras.models import (
     ExportTemplate,
     GitRepository,
     Job,
-    JobLogEntry,
     JobResult,
     Role,
     Tag,
 )
 from nautobot.extras.registry import DatasourceContent, register_datasource_contents
-from nautobot.extras.utils import jobs_in_directory, refresh_job_model_from_job_class
+from nautobot.extras.utils import refresh_job_model_from_job_class
 from nautobot.tenancy.models import TenantGroup, Tenant
 from nautobot.virtualization.models import ClusterGroup, Cluster, VirtualMachine
-from .registry import refresh_datasource_content
 from .utils import files_from_contenttype_directories
 
 
@@ -54,31 +50,29 @@ GitJobResult = namedtuple("GitJobResult", ["job_result", "repository_record"])
 GitRepoInfo = namedtuple("GitRepoInfo", ["from_url", "to_path", "from_branch"])
 
 
-def enqueue_git_repository_helper(repository, request, func, **kwargs):
+def enqueue_git_repository_helper(repository, user, job_class, **kwargs):
     """
     Wrapper for JobResult.enqueue_job() to enqueue one of several possible Git repository functions.
     """
-    git_repository_content_type = ContentType.objects.get_for_model(GitRepository)
-    JobResult.enqueue_job(
-        func,
-        repository.name,
-        git_repository_content_type,
-        request.user,
-        repository_pk=repository.pk,
-        request=copy_safe_request(request),
-    )
+    job_model = job_class().job_model
 
+    return JobResult.enqueue_job(job_model, user, repository=repository.pk)
 
-def enqueue_git_repository_diff_origin_and_local(repository, request):
+
+def enqueue_git_repository_diff_origin_and_local(repository, user):
     """Convenience wrapper for JobResult.enqueue_job() to enqueue the git_repository_diff_origin_and_local job."""
-    enqueue_git_repository_helper(repository, request, git_repository_diff_origin_and_local)
+    from nautobot.core.jobs import GitRepositoryDryRun
+
+    return enqueue_git_repository_helper(repository, user, GitRepositoryDryRun)
 
 
-def enqueue_pull_git_repository_and_refresh_data(repository, request):
+def enqueue_pull_git_repository_and_refresh_data(repository, user):
     """
     Convenience wrapper for JobResult.enqueue_job() to enqueue the pull_git_repository_and_refresh_data job.
     """
-    enqueue_git_repository_helper(repository, request, pull_git_repository_and_refresh_data)
+    from nautobot.core.jobs import GitRepositorySync
+
+    return enqueue_git_repository_helper(repository, user, GitRepositorySync)
 
 
 def get_job_result_and_repository_record(repository_pk, job_result_pk, logger):  # pylint: disable=redefined-outer-name
@@ -97,121 +91,14 @@ def get_job_result_and_repository_record(repository_pk, job_result_pk, logger):
     if not repository_record:
         job_result.log(
             f"No GitRepository {repository_pk} found!",
-            level_choice=LogLevelChoices.LOG_FAILURE,
+            level_choice=LogLevelChoices.LOG_ERROR,
             logger=logger,
         )
-        job_result.set_status(JobResultStatusChoices.STATUS_FAILURE)
-        job_result.save()
         return GitJobResult(job_result=job_result, repository_record=None)
 
     return GitJobResult(job_result=job_result, repository_record=repository_record)
 
 
-# TODO(jathan): This should likely be deleted since it's coercing state and this
-# should be managed by the database backend.
-def log_job_result_final_status(job_result, job_type):
-    """Check Job status and save log to DB
-    Args:
-        job_result (JobResult): JobResult Instance
-        job_type (str): job type which is used in log message, e.g dry run/synchronization etc.
-    """
-    if job_result.status not in JobResultStatusChoices.READY_STATES:
-        if JobLogEntry.objects.filter(job_result__pk=job_result.pk, log_level=LogLevelChoices.LOG_FAILURE).exists():
-            job_result.set_status(JobResultStatusChoices.STATUS_FAILURE)
-        else:
-            job_result.set_status(JobResultStatusChoices.STATUS_SUCCESS)
-    job_result.log(
-        f"Repository {job_type} completed in {job_result.duration}",
-        level_choice=LogLevelChoices.LOG_INFO,
-        logger=logger,
-    )
-    job_result.save()
-
-
-# TODO(jathan): The state transition stuff here should be deleted in exchange
-# for trusting the database backend.
-@nautobot_task
-def pull_git_repository_and_refresh_data(repository_pk, request, job_result_pk):
-    """
-    Worker function to clone and/or pull a Git repository into Nautobot, then invoke refresh_datasource_content().
-    """
-    job_result, repository_record = get_job_result_and_repository_record(
-        repository_pk=repository_pk,
-        job_result_pk=job_result_pk,
-        logger=logger,
-    )
-
-    if not repository_record:
-        return
-
-    job_result.log(f'Creating/refreshing local copy of Git repository "{repository_record.name}"...', logger=logger)
-    job_result.set_status(JobResultStatusChoices.STATUS_STARTED)
-    job_result.save()
-
-    try:
-        if not os.path.exists(settings.GIT_ROOT):
-            os.makedirs(settings.GIT_ROOT)
-
-        ensure_git_repository(
-            repository_record,
-            job_result=job_result,
-            logger=logger,
-        )
-
-        job_result.log(
-            f'The current Git repository hash is "{repository_record.current_head}"',
-            level_choice=LogLevelChoices.LOG_INFO,
-            logger=logger,
-        )
-
-        refresh_datasource_content("extras.gitrepository", repository_record, request, job_result, delete=False)
-
-    except Exception as exc:
-        job_result.log(
-            f"Error while refreshing {repository_record.name}: {exc}",
-            level_choice=LogLevelChoices.LOG_FAILURE,
-        )
-        job_result.set_status(JobResultStatusChoices.STATUS_FAILURE)
-
-    finally:
-        log_job_result_final_status(job_result, "synchronization")
-
-
-# TODO(jathan): The state transition stuff here should be deleted in exchange
-# for trusting the database backend.
-@nautobot_task
-def git_repository_diff_origin_and_local(repository_pk, request, job_result_pk, **kwargs):
-    """
-    Worker function to run a dry run on a Git repository.
-    """
-    job_result, repository_record = get_job_result_and_repository_record(
-        repository_pk,
-        job_result_pk,
-        logger=logger,
-    )
-    if not repository_record:
-        return
-
-    job_result.log(f'Running a Dry Run on Git repository "{repository_record.name}"...', logger=logger)
-    job_result.set_status(JobResultStatusChoices.STATUS_STARTED)
-    job_result.save()
-    try:
-        if not os.path.exists(settings.GIT_ROOT):
-            os.makedirs(settings.GIT_ROOT)
-
-        git_repository_dry_run(repository_record, job_result=job_result, logger=logger)
-
-    except Exception as exc:
-        job_result.log(
-            f"Error while running a dry run on {repository_record.name}: {exc}",
-            level_choice=LogLevelChoices.LOG_FAILURE,
-        )
-        job_result.set_status(JobResultStatusChoices.STATUS_FAILURE)
-
-    finally:
-        log_job_result_final_status(job_result, "dry run")
-
-
 def get_repo_from_url_to_path_and_from_branch(repository_record):
     """Returns the from_url, to_path and from_branch of a Git Repo
     Returns:
@@ -266,43 +153,58 @@ def ensure_git_repository(
     repository_record, job_result=None, logger=None, head=None  # pylint: disable=redefined-outer-name
 ):
     """Ensure that the given Git repo is present, up-to-date, and has the correct branch selected.
-    Note that this function may be called independently of the `pull_git_repository_and_refresh_data` job,
+
+    Note that this function may be called independently of the `GitRepositorySync` job,
     such as to ensure that different Nautobot instances and/or worker instances all have a local copy of the same HEAD.
+
     Args:
       repository_record (GitRepository): Repository to ensure the state of.
       job_result (JobResult): Optional JobResult to store results into.
       logger (logging.Logger): Optional Logger to additionally log results to.
       head (str): Optional Git commit hash to check out instead of pulling branch latest.
+
+    Returns:
+      bool: Whether any change to the local repo actually occurred.
     """
 
     from_url, to_path, from_branch = get_repo_from_url_to_path_and_from_branch(repository_record)
 
     try:
         repo_helper = GitRepo(to_path, from_url)
-        head = repo_helper.checkout(from_branch, head)
+        head, changed = repo_helper.checkout(from_branch, head)
         if repository_record.current_head != head:
             repository_record.current_head = head
-            # Make sure we don't recursively trigger a new resync of the repository!
-            repository_record.save(trigger_resync=False)
+            repository_record.save()
 
+    # FIXME(jathan): As a part of jobs overhaul, this error-handling should be removed since this
+    # should only ever be called in the context of a Git sync job. Also, all logging directly from a
+    # JobResult should also be replaced with just trusting the logger to do the correct thing (such
+    # as from the Job class).
     except Exception as exc:
         if job_result:
-            job_result.set_status(JobResultStatusChoices.STATUS_FAILURE)
-            job_result.log(str(exc), level_choice=LogLevelChoices.LOG_FAILURE, logger=logger)
-            job_result.save()
+            job_result.log(str(exc), level_choice=LogLevelChoices.LOG_ERROR, logger=logger)
         elif logger:
             logger.error(str(exc))
         raise
 
     if job_result:
+        if changed:
+            job_result.log(
+                "Repository successfully refreshed",
+                level_choice=LogLevelChoices.LOG_INFO,
+                logger=logger,
+            )
         job_result.log(
-            "Repository successfully refreshed",
-            level_choice=LogLevelChoices.LOG_SUCCESS,
+            f'The current Git repository hash is "{repository_record.current_head}"',
+            level_choice=LogLevelChoices.LOG_INFO,
             logger=logger,
         )
-        job_result.save()
     elif logger:
-        logger.info("Repository successfully refreshed")
+        if changed:
+            logger.info("Repository successfully refreshed")
+        logger.info(f'The current Git repository hash is "{repository_record.current_head}"')
+
+    return changed
 
 
 def git_repository_dry_run(repository_record, job_result=None, logger=None):  # pylint: disable=redefined-outer-name
@@ -328,15 +230,13 @@ def git_repository_dry_run(repository_record, job_result=None, logger=None):  #
 
     except Exception as exc:
         if job_result:
-            job_result.set_status(JobResultStatusChoices.STATUS_FAILURE)
-            job_result.log(str(exc), level_choice=LogLevelChoices.LOG_FAILURE, logger=logger)
-            job_result.save()
+            job_result.log(str(exc), level_choice=LogLevelChoices.LOG_ERROR, logger=logger)
         elif logger:
             logger.error(str(exc))
         raise
 
     if job_result:
-        job_result.log("Repository dry run successful", level_choice=LogLevelChoices.LOG_SUCCESS, logger=logger)
+        job_result.log("Repository dry run successful", level_choice=LogLevelChoices.LOG_INFO, logger=logger)
     elif logger:
         logger.info("Repository dry run successful")
 
@@ -392,11 +292,10 @@ def update_git_config_contexts(repository_record, job_result):
         except Exception as exc:
             job_result.log(
                 f"Error in loading config context data from `{file_name}`: {exc}",
-                level_choice=LogLevelChoices.LOG_FAILURE,
+                level_choice=LogLevelChoices.LOG_ERROR,
                 grouping="config contexts",
                 logger=logger,
             )
-            job_result.save()
 
     # Next, handle the "filter/slug directory structure case - files in <filter_type>/<slug>.(json|yaml)
     for filter_type in (
@@ -446,11 +345,10 @@ def update_git_config_contexts(repository_record, job_result):
             except Exception as exc:
                 job_result.log(
                     f"Error in loading config context data from `{file_name}`: {exc}",
-                    level_choice=LogLevelChoices.LOG_FAILURE,
+                    level_choice=LogLevelChoices.LOG_ERROR,
                     grouping="config contexts",
                     logger=logger,
                 )
-                job_result.save()
 
     # Finally, handle device- and virtual-machine-specific "local" context in (devices|virtual_machines)/<name>.(json|yaml)
     for local_type in ("devices", "virtual_machines"):
@@ -490,11 +388,10 @@ def update_git_config_contexts(repository_record, job_result):
             except Exception as exc:
                 job_result.log(
                     f"Error in loading local config context from `{local_type}/{file_name}`: {exc}",
-                    level_choice=LogLevelChoices.LOG_FAILURE,
+                    level_choice=LogLevelChoices.LOG_ERROR,
                     grouping="local config contexts",
                     logger=logger,
                 )
-                job_result.save()
 
     # Delete any prior contexts that are owned by this repository but were not created/updated above
     delete_git_config_contexts(
@@ -616,7 +513,7 @@ def import_config_context(context_data, repository_record, job_result, logger):
                     job_result.log(
                         f"ConfigContextSchema {context_metadata['config_context_schema']} does not exist.",
                         obj=context_record,
-                        level_choice=LogLevelChoices.LOG_FAILURE,
+                        level_choice=LogLevelChoices.LOG_ERROR,
                         grouping="config contexts",
                         logger=logger,
                     )
@@ -650,7 +547,7 @@ def import_config_context(context_data, repository_record, job_result, logger):
         job_result.log(
             "Successfully created config context",
             obj=context_record,
-            level_choice=LogLevelChoices.LOG_SUCCESS,
+            level_choice=LogLevelChoices.LOG_INFO,
             grouping="config contexts",
             logger=logger,
         )
@@ -658,7 +555,7 @@ def import_config_context(context_data, repository_record, job_result, logger):
         job_result.log(
             "Successfully refreshed config context",
             obj=context_record,
-            level_choice=LogLevelChoices.LOG_SUCCESS,
+            level_choice=LogLevelChoices.LOG_INFO,
             grouping="config contexts",
             logger=logger,
         )
@@ -699,7 +596,7 @@ def import_local_config_context(
         job_result.log(
             f"DATA CONFLICT: Local context data is owned by another owner, {record.local_config_context_data_owner}",
             obj=record,
-            level_choice=LogLevelChoices.LOG_FAILURE,
+            level_choice=LogLevelChoices.LOG_ERROR,
             grouping="local config contexts",
             logger=logger,
         )
@@ -722,7 +619,7 @@ def import_local_config_context(
     job_result.log(
         "Successfully updated local config context",
         obj=record,
-        level_choice=LogLevelChoices.LOG_SUCCESS,
+        level_choice=LogLevelChoices.LOG_INFO,
         grouping="local config contexts",
         logger=logger,
     )
@@ -821,11 +718,10 @@ def update_git_config_context_schemas(repository_record, job_result):
         except Exception as exc:
             job_result.log(
                 f"Error in loading config context schema data from `{file_name}`: {exc}",
-                level_choice=LogLevelChoices.LOG_FAILURE,
+                level_choice=LogLevelChoices.LOG_ERROR,
                 grouping="config context schemas",
                 logger=logger,
             )
-            job_result.save()
 
     # Delete any prior contexts that are owned by this repository but were not created/updated above
     delete_git_config_context_schemas(
@@ -880,7 +776,7 @@ def import_config_context_schema(
         job_result.log(
             "Successfully created config context schema",
             obj=schema_record,
-            level_choice=LogLevelChoices.LOG_SUCCESS,
+            level_choice=LogLevelChoices.LOG_INFO,
             grouping="config context schemas",
             logger=logger,
         )
@@ -889,7 +785,7 @@ def import_config_context_schema(
         job_result.log(
             "Successfully refreshed config context schema",
             obj=schema_record,
-            level_choice=LogLevelChoices.LOG_SUCCESS,
+            level_choice=LogLevelChoices.LOG_INFO,
             grouping="config context schemas",
             logger=logger,
         )
@@ -927,34 +823,77 @@ def delete_git_config_context_schemas(repository_record, job_result, preserve=()
 #
 
 
+def refresh_code_from_repository(repository_slug, consumer=None, skip_reimport=False):
+    """
+    After cloning/updating a GitRepository on disk, call this function to reload and reregister the repo's Python code.
+
+    Args:
+        repository_slug (str): Repository directory in GIT_ROOT that was refreshed.
+        consumer (celery.worker.Consumer): Celery Consumer to update as well
+        skip_reimport (bool): If True, unload existing code from this repository but do not re-import it.
+    """
+    if settings.GIT_ROOT not in sys.path:
+        sys.path.append(settings.GIT_ROOT)
+
+    app = consumer.app if consumer is not None else celery_app
+    # TODO: This is ugly, but when app.use_fast_trace_task is set (true by default), Celery calls
+    # celery.app.trace.fast_trace_task(...) which assumes that all tasks are cached and have a valid `__trace__()`
+    # function defined. In theory consumer.update_strategies() (below) should ensure this, but it doesn't
+    # go far enough (possibly a discrepancy between the main worker process and the prefork executors?)
+    # as we can and do still encounter errors where `task.__trace__` is unexpectedly None.
+    # For now, simply disabling use_fast_trace_task forces the task trace function to be rebuilt each time,
+    # which avoids the issue at the cost of very slight overhead.
+    app.use_fast_trace_task = False
+
+    # Unload any previous version of this module and its submodules if present
+    for module_name in list(sys.modules):
+        if module_name == repository_slug or module_name.startswith(f"{repository_slug}."):
+            logger.debug("Unloading module %s", module_name)
+            if module_name in app.loader.task_modules:
+                app.loader.task_modules.remove(module_name)
+            del sys.modules[module_name]
+
+    # Unregister any previous Celery tasks from this module
+    for task_name in list(app.tasks):
+        if task_name.startswith(f"{repository_slug}."):
+            logger.debug("Unregistering Celery task %s", task_name)
+            app.tasks.unregister(task_name)
+            if consumer is not None:
+                del consumer.strategies[task_name]
+
+    if not skip_reimport:
+        try:
+            repository = GitRepository.objects.get(slug=repository_slug)
+            if "extras.job" in repository.provided_contents:
+                # Re-import Celery tasks from this module
+                logger.debug("Importing Jobs from %s.jobs in GIT_ROOT", repository_slug)
+                app.loader.import_task_module(f"{repository_slug}.jobs")
+                if consumer is not None:
+                    consumer.update_strategies()
+        except GitRepository.DoesNotExist as exc:
+            logger.error("Unable to reload Jobs from %s.jobs: %s", repository_slug, exc)
+            raise
+
+
 def refresh_git_jobs(repository_record, job_result, delete=False):
     """Callback function for GitRepository updates - refresh all Job records managed by this repository."""
     installed_jobs = []
     if "extras.job" in repository_record.provided_contents and not delete:
-        jobs_path = os.path.join(repository_record.filesystem_path, "jobs")
-        if os.path.isdir(jobs_path):
-            for job_info in jobs_in_directory(jobs_path, report_errors=True):
-                if job_info.error is not None:
-                    job_result.log(
-                        message=f"Error in loading Jobs from `{job_info.module_name}`: `{job_info.error}`",
-                        grouping="jobs",
-                        level_choice=LogLevelChoices.LOG_FAILURE,
-                        logger=logger,
-                    )
-                    continue
+        found_jobs = False
+        try:
+            refresh_code_from_repository(repository_record.slug)
 
-                job_model, created = refresh_job_model_from_job_class(
-                    Job,
-                    JobSourceChoices.SOURCE_GIT,
-                    job_info.job_class,
-                    git_repository=repository_record,
-                )
+            for task_name, task in celery_app.tasks.items():
+                if not task_name.startswith(f"{repository_record.slug}."):
+                    continue
+                found_jobs = True
+                job_model, created = refresh_job_model_from_job_class(Job, task.__class__)
 
                 if job_model is None:
                     job_result.log(
                         message="Failed to create Job record; check Nautobot logs for details",
                         grouping="jobs",
-                        level_choice=LogLevelChoices.LOG_FAILURE,
+                        level_choice=LogLevelChoices.LOG_ERROR,
                         logger=logger,
                     )
                     continue
@@ -967,19 +906,30 @@ def refresh_git_jobs(repository_record, job_result, delete=False):
                     message=message,
                     obj=job_model,
                     grouping="jobs",
-                    level_choice=LogLevelChoices.LOG_SUCCESS,
+                    level_choice=LogLevelChoices.LOG_INFO,
                     logger=logger,
                 )
                 installed_jobs.append(job_model)
-        else:
+
+            if not found_jobs:
+                job_result.log(
+                    "No jobs were registered on loading the `jobs` submodule. Did you miss a `register_jobs()` call?",
+                    grouping="jobs",
+                    level_choice=LogLevelChoices.LOG_WARNING,
+                    logger=logger,
+                )
+        except Exception as exc:
             job_result.log(
-                "No `jobs` subdirectory found in Git repository",
+                f"Error in loading Jobs from Git repository: {exc}",
                 grouping="jobs",
-                level_choice=LogLevelChoices.LOG_WARNING,
+                level_choice=LogLevelChoices.LOG_ERROR,
                 logger=logger,
             )
+    else:
+        # Unload code from this repository, do not reimport it
+        refresh_code_from_repository(repository_record.slug, skip_reimport=True)
 
-    for job_model in Job.objects.filter(source=JobSourceChoices.SOURCE_GIT, git_repository=repository_record):
+    for job_model in Job.objects.filter(module_name__startswith=f"{repository_record.slug}."):
         if job_model.installed and job_model not in installed_jobs:
             job_result.log(
                 message="Marking Job record as no longer installed",
@@ -1093,7 +1043,7 @@ def update_git_export_templates(repository_record, job_result):
                 job_result.log(
                     "Successfully created export template",
                     obj=template_record,
-                    level_choice=LogLevelChoices.LOG_SUCCESS,
+                    level_choice=LogLevelChoices.LOG_INFO,
                     grouping="export templates",
                     logger=logger,
                 )
@@ -1101,7 +1051,7 @@ def update_git_export_templates(repository_record, job_result):
                 job_result.log(
                     "Successfully refreshed export template",
                     obj=template_record,
-                    level_choice=LogLevelChoices.LOG_SUCCESS,
+                    level_choice=LogLevelChoices.LOG_INFO,
                     grouping="export templates",
                     logger=logger,
                 )
@@ -1118,11 +1068,10 @@ def update_git_export_templates(repository_record, job_result):
             job_result.log(
                 str(exc),
                 obj=template_record,
-                level_choice=LogLevelChoices.LOG_FAILURE,
+                level_choice=LogLevelChoices.LOG_ERROR,
                 grouping="export templates",
                 logger=logger,
             )
-            job_result.save()
 
     # Delete any prior templates that are owned by this repository but were not discovered above
     delete_git_export_templates(repository_record, job_result, preserve=managed_export_templates)

nautobot/extras/datasources/registry.py

@@ -1,7 +1,6 @@
 """Registry-related APIs for datasources."""
 
-from nautobot.extras.choices import JobResultStatusChoices, LogLevelChoices
-from nautobot.extras.context_managers import change_logging, JobChangeContext
+from nautobot.extras.choices import LogLevelChoices
 from nautobot.extras.registry import registry
 
 
@@ -19,7 +18,7 @@ def get_datasource_content_choices(model_name):
     )
 
 
-def refresh_datasource_content(model_name, record, request, job_result, delete=False):
+def refresh_datasource_content(model_name, record, user, job_result, delete=False):
     """Invoke the refresh callbacks for every content type registered for this model.
 
     Note that these callback functions are invoked regardless of whether a given model instance actually is flagged
@@ -29,39 +28,30 @@ def refresh_datasource_content(model_name, record, request, job_result, delete=F
     Args:
         model_name (str): Identifier of the datasource owner, such as "extras.gitrepository"
         record (models.Model): Datasource model instance, such as a GitRepository record
-        request (HttpRequest): Initiating request for this refresh, optional, used for change logging if provided
+        user (User): Initiating user for this refresh, optional, used for change logging if provided
         job_result (JobResult): Passed through to the callback functions to use with logging their actions.
         delete (bool): True if the record is being deleted; False if it is being created/updated.
     """
     job_result.log(f"Refreshing data provided by {record}...", level_choice=LogLevelChoices.LOG_INFO)
-    job_result.save()
-    if request:
-        change_context = JobChangeContext(user=request.user)
-        # TODO(jathan): Review how we can eliminate these deliberate `set_status()` and `save()`
-        # calls,moving them to Celery-friendly primitives instead.
-        with change_logging(change_context):
-            for entry in get_datasource_contents(model_name):
-                job_result.log(f"Refreshing {entry.name}...", level_choice=LogLevelChoices.LOG_INFO)
-                try:
-                    entry.callback(record, job_result, delete=delete)
-                except Exception as exc:
-                    job_result.log(
-                        f"Error while refreshing {entry.name}: {exc}", level_choice=LogLevelChoices.LOG_FAILURE
-                    )
-                    job_result.set_status(JobResultStatusChoices.STATUS_FAILURE)
-                job_result.save()
-            job_result.log(f"Data refresh from {record} complete!", level_choice=LogLevelChoices.LOG_INFO)
-            job_result.save()
-    else:
-        # TODO(jathan): Review how we can eliminate these deliberate `set_status()` and `save()`
-        # calls,moving them to Celery-friendly primitives instead.
-        for entry in get_datasource_contents(model_name):
-            job_result.log(f"Refreshing {entry.name}...", level_choice=LogLevelChoices.LOG_INFO)
-            try:
-                entry.callback(record, job_result, delete=delete)
-            except Exception as exc:
-                job_result.log(f"Error while refreshing {entry.name}: {exc}", level_choice=LogLevelChoices.LOG_FAILURE)
-                job_result.set_status(JobResultStatusChoices.STATUS_FAILURE)
-            job_result.save()
-        job_result.log(f"Data refresh from {record} complete!", level_choice=LogLevelChoices.LOG_INFO)
-        job_result.save()
+
+    for entry in get_datasource_contents(model_name):
+        job_result.log(f"Refreshing {entry.name}...", level_choice=LogLevelChoices.LOG_INFO)
+        try:
+            entry.callback(record, job_result, delete=delete)
+        except Exception as exc:
+            job_result.log(f"Error while refreshing {entry.name}: {exc}", level_choice=LogLevelChoices.LOG_ERROR)
+            raise
+
+    # Now that any exception will fail a Job and Git Repository syncs are jobs,
+    # and we also cannot micro-manage the JobResult state, we had to add a final
+    # check here to ensure that any failed log events by the various content
+    # callbacks will result in this task "failing successfully" by raising an
+    # exception.
+    failure_logs = job_result.job_log_entries.filter(log_level=LogLevelChoices.LOG_ERROR)
+    if failure_logs.exists():
+        msg = f"Failed to refresh data provided by {record}. Please see logs."
+        job_result.log(msg, level_choice=LogLevelChoices.LOG_ERROR)
+        raise RuntimeError(msg)
+
+    # Otherwise, log a friendly info message.
+    job_result.log(f"Data refresh from {record} complete!", level_choice=LogLevelChoices.LOG_INFO)