pulpcore 3.89.1__py3-none-any.whl → 3.90.0__py3-none-any.whl

pulpcore/app/viewsets/base.py

@@ -13,7 +13,7 @@ from rest_framework import viewsets
 from rest_framework.decorators import action
 from rest_framework.generics import get_object_or_404
 from rest_framework.response import Response
-from pulpcore.openapi import PulpAutoSchema
+from pulpcore.openapi import PulpAutoSchema, InheritSerializer
 from rest_framework.serializers import ValidationError as DRFValidationError, ListField, CharField
 
 from pulpcore.app import tasks
@@ -482,27 +482,31 @@ class AsyncUpdateMixin(AsyncReservedObjectMixin):
     ALLOW_NON_BLOCKING_UPDATE = True
 
     @extend_schema(
-        description="Trigger an asynchronous update task",
-        responses={202: AsyncOperationResponseSerializer},
+        description="Update the entity and trigger an asynchronous task if necessary",
+        responses={200: InheritSerializer, 202: AsyncOperationResponseSerializer},
     )
     def update(self, request, pk, **kwargs):
         partial = kwargs.pop("partial", False)
         instance = self.get_object()
         serializer = self.get_serializer(instance, data=request.data, partial=partial)
         serializer.is_valid(raise_exception=True)
-        app_label = instance._meta.app_label
-        task = dispatch(
-            tasks.base.ageneral_update,
-            exclusive_resources=self.async_reserved_resources(instance),
-            args=(pk, app_label, serializer.__class__.__name__),
-            kwargs={"data": request.data, "partial": partial},
-            immediate=self.ALLOW_NON_BLOCKING_UPDATE,
-        )
-        return OperationPostponedResponse(task, request)
+
+        if all(getattr(instance, key) == value for key, value in serializer.validated_data.items()):
+            return Response(serializer.data)
+        else:
+            app_label = instance._meta.app_label
+            task = dispatch(
+                tasks.base.ageneral_update,
+                exclusive_resources=self.async_reserved_resources(instance),
+                args=(pk, app_label, serializer.__class__.__name__),
+                kwargs={"data": request.data, "partial": partial},
+                immediate=self.ALLOW_NON_BLOCKING_UPDATE,
+            )
+            return OperationPostponedResponse(task, request)
 
     @extend_schema(
-        description="Trigger an asynchronous partial update task",
-        responses={202: AsyncOperationResponseSerializer},
+        description="Update the entity partially and trigger an asynchronous task if necessary",
+        responses={200: InheritSerializer, 202: AsyncOperationResponseSerializer},
     )
     def partial_update(self, request, *args, **kwargs):
         kwargs["partial"] = True

pulpcore/app/viewsets/domain.py

@@ -109,7 +109,7 @@ class DomainViewSet(
 
     @extend_schema(
         description="Trigger an asynchronous update task",
-        responses={202: AsyncOperationResponseSerializer},
+        responses={200: DomainSerializer, 202: AsyncOperationResponseSerializer},
     )
     def update(self, request, pk, **kwargs):
         """Prevent trying to update the default domain."""

pulpcore/app/viewsets/exporter.py

@@ -1,6 +1,5 @@
-from django.conf import settings
 from drf_spectacular.utils import extend_schema
-from rest_framework import mixins, exceptions
+from rest_framework import mixins
 
 from pulpcore.app.models import (
     Export,
@@ -34,8 +33,6 @@ from pulpcore.app.viewsets.base import NAME_FILTER_OPTIONS
 from pulpcore.plugin.tasking import dispatch
 from pulpcore.app.response import OperationPostponedResponse
 
-from gettext import gettext as _
-
 
 class ExporterViewSet(
     NamedModelViewSet,
@@ -120,8 +117,6 @@ class PulpExportViewSet(ExportViewSet):
         """
         Generates a Task to export the set of repositories assigned to a specific PulpExporter.
         """
-        if settings.DOMAIN_ENABLED:
-            raise exceptions.ValidationError(_("Export not supported with Domains enabled."))
         # Validate Exporter
         exporter = PulpExporter.objects.get(pk=exporter_pk).cast()
         ExporterSerializer.validate_path(exporter.path, check_is_dir=True)
@@ -159,8 +154,6 @@ class FilesystemExportViewSet(ExportViewSet):
         """
         Generates a Task to export files to the filesystem.
         """
-        if settings.DOMAIN_ENABLED:
-            raise exceptions.ValidationError(_("Export not supported with Domains enabled."))
         # Validate Exporter
         exporter = FilesystemExporter.objects.get(pk=exporter_pk).cast()
         ExporterSerializer.validate_path(exporter.path, check_is_dir=True)

pulpcore/app/viewsets/importer.py

@@ -1,7 +1,6 @@
-from django.conf import settings
 from django.http import Http404
 from drf_spectacular.utils import extend_schema
-from rest_framework import mixins, exceptions
+from rest_framework import mixins
 
 from pulpcore.app.models import (
     Import,
@@ -25,8 +24,6 @@ from pulpcore.app.viewsets import (
 from pulpcore.app.viewsets.base import NAME_FILTER_OPTIONS
 from pulpcore.tasking.tasks import dispatch
 
-from gettext import gettext as _
-
 
 class ImporterViewSet(
     NamedModelViewSet,
@@ -87,8 +84,6 @@ class PulpImportViewSet(ImportViewSet):
     )
     def create(self, request, importer_pk):
         """Import a Pulp export into Pulp."""
-        if settings.DOMAIN_ENABLED:
-            raise exceptions.ValidationError(_("Import not supported with Domains enabled."))
         try:
             importer = PulpImporter.objects.get(pk=importer_pk)
         except PulpImporter.DoesNotExist:

pulpcore/app/viewsets/task.py

@@ -68,7 +68,6 @@ class TaskFilter(BaseFilterSet):
         model = Task
         fields = {
             "state": ["exact", "in", "ne"],
-            "worker": ["exact", "in", "isnull"],
             "name": ["exact", "contains", "in", "ne"],
             "logging_cid": ["exact", "contains"],
             "pulp_created": DATETIME_FILTER_OPTIONS,

pulpcore/openapi/__init__.py

@@ -23,8 +23,8 @@ from drf_spectacular.plumbing import (
 )
 from drf_spectacular.settings import spectacular_settings
 from drf_spectacular.types import OpenApiTypes
-from drf_spectacular.utils import OpenApiParameter, extend_schema_field
-from drf_spectacular.extensions import OpenApiAuthenticationExtension
+from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_field
+from drf_spectacular.extensions import OpenApiViewExtension, OpenApiAuthenticationExtension
 from rest_framework import mixins, serializers
 from rest_framework.exceptions import ParseError
 from rest_framework.request import Request
@@ -48,6 +48,10 @@ API_ROOT_NO_FRONT_SLASH = API_ROOT_NO_FRONT_SLASH.replace("<", "{").replace(">",
 extend_schema_field(OpenApiTypes.INT64)(serializers.IntegerField)
 
 
+class InheritSerializer(serializers.Serializer):
+    """This is a dummy tracer to allow mixins to refer to the natural serializer of a viewset."""
+
+
 class PulpAutoSchema(AutoSchema):
     """Pulp Auto Schema."""
 
@@ -238,6 +242,7 @@ class PulpAutoSchema(AutoSchema):
         """
         Handle response status code.
         """
+        # DRF handles this most of the time. But it seems set_label still needs it.
         response = super()._get_response_bodies()
         if (
             self.method == "POST"
@@ -248,6 +253,15 @@ class PulpAutoSchema(AutoSchema):
 
         return response
 
+    def _get_response_for_code(
+        self, serializer, status_code, media_types=None, direction="response"
+    ):
+        """Hack to replace the InheritSerializer with the real deal."""
+        if serializer == InheritSerializer:
+            serializer = self._get_serializer()
+
+        return super()._get_response_for_code(serializer, status_code, media_types, direction)
+
 
 class PulpSchemaGenerator(SchemaGenerator):
     """Pulp Schema Generator."""

pulpcore/plugin/tasking.py

@@ -1,5 +1,5 @@
 # Support plugins dispatching tasks
-from pulpcore.tasking.tasks import dispatch
+from pulpcore.tasking.tasks import dispatch, adispatch
 
 from pulpcore.app.tasks import (
     ageneral_update,
@@ -13,13 +13,14 @@ from pulpcore.app.tasks import (
     reclaim_space,
 )
 from pulpcore.app.tasks.vulnerability_report import check_content
-from pulpcore.app.tasks.repository import add_and_remove
+from pulpcore.app.tasks.repository import add_and_remove, aadd_and_remove
 
 
 __all__ = [
     "ageneral_update",
     "check_content",
     "dispatch",
+    "adispatch",
     "fs_publication_export",
     "fs_repo_version_export",
     "general_create",
@@ -29,4 +30,5 @@ __all__ = [
     "orphan_cleanup",
     "reclaim_space",
     "add_and_remove",
+    "aadd_and_remove",
 ]

pulpcore/tasking/tasks.py

@@ -6,8 +6,9 @@ import os
 import sys
 import traceback
 import tempfile
-import threading
 from gettext import gettext as _
+from contextlib import contextmanager
+from asgiref.sync import sync_to_async, async_to_sync
 
 from django.conf import settings
 from django.db import connection
@@ -60,91 +61,129 @@ def execute_task(task):
     contextvars.copy_context().run(_execute_task, task)
 
 
+async def aexecute_task(task):
+    # This extra stack is needed to isolate the current_task ContextVar
+    await contextvars.copy_context().run(_aexecute_task, task)
+
+
 def _execute_task(task):
     # Store the task id in the context for `Task.current()`.
     current_task.set(task)
     task.set_running()
     domain = get_domain()
     try:
-        _logger.info(
-            "Starting task id: %s in domain: %s, task_type: %s, immediate: %s, deferred: %s",
-            task.pk,
-            domain.name,
-            task.name,
-            str(task.immediate),
-            str(task.deferred),
-        )
-
-        # Execute task
-        module_name, function_name = task.name.rsplit(".", 1)
-        module = importlib.import_module(module_name)
-        func = getattr(module, function_name)
-        args = task.enc_args or ()
-        kwargs = task.enc_kwargs or {}
-        immediate = task.immediate
-        is_coroutine_fn = asyncio.iscoroutinefunction(func)
-
-        if immediate and not is_coroutine_fn:
-            raise ValueError("Immediate tasks must be async functions.")
-
-        if is_coroutine_fn:
-            # both regular and immediate tasks can be coroutines, but only immediate must timeout
-            _logger.debug("Task is coroutine %s", task.pk)
-            coro = func(*args, **kwargs)
-            if immediate:
-                coro = asyncio.wait_for(coro, timeout=IMMEDIATE_TIMEOUT)
-            loop = asyncio.get_event_loop()
-            try:
-                result = loop.run_until_complete(coro)
-            except asyncio.TimeoutError:
-                _logger.info(
-                    "Immediate task %s timed out after %s seconds.", task.pk, IMMEDIATE_TIMEOUT
-                )
-                raise RuntimeError(
-                    "Immediate task timed out after {timeout} seconds.".format(
-                        timeout=IMMEDIATE_TIMEOUT,
-                    )
-                )
-        else:
-            result = func(*args, **kwargs)
-
+        log_task_start(task, domain)
+        task_function = get_task_function(task)
+        result = task_function()
     except Exception:
         exc_type, exc, tb = sys.exc_info()
         task.set_failed(exc, tb)
-        _logger.info(
-            "Task[{task_type}] {task_pk} failed ({exc_type}: {exc}) in domain: {domain}".format(
-                task_type=task.name,
-                task_pk=task.pk,
-                exc_type=exc_type.__name__,
-                exc=exc,
-                domain=domain.name,
-            )
-        )
-        _logger.info("\n".join(traceback.format_list(traceback.extract_tb(tb))))
+        log_task_failed(task, exc_type, exc, tb, domain)
         send_task_notification(task)
     else:
         task.set_completed(result)
-        execution_time = task.finished_at - task.started_at
-        execution_time_us = int(execution_time.total_seconds() * 1_000_000)  # μs
-        _logger.info(
-            "Task completed %s in domain:"
-            " %s, task_type: %s, immediate: %s, deferred: %s, execution_time: %s μs",
-            task.pk,
-            domain.name,
-            task.name,
-            str(task.immediate),
-            str(task.deferred),
-            execution_time_us,
-        )
+        log_task_completed(task, domain)
         send_task_notification(task)
+        return result
+    return None
 
 
-def running_from_thread_pool() -> bool:
-    # TODO: this needs an alternative approach ASAP!
-    # Currently we rely on the weak fact that ThreadPoolExecutor names threads like:
-    # "ThreadPoolExecutor-0_0"
-    thread_name = threading.current_thread().name
-    return "ThreadPoolExecutor" in thread_name
+async def _aexecute_task(task):
+    # Store the task id in the context for `Task.current()`.
+    current_task.set(task)
+    await sync_to_async(task.set_running)()
+    domain = get_domain()
+    try:
+        coroutine = get_task_function(task, ensure_coroutine=True)
+        result = await coroutine
+    except Exception:
+        exc_type, exc, tb = sys.exc_info()
+        await sync_to_async(task.set_failed)(exc, tb)
+        log_task_failed(task, exc_type, exc, tb, domain)
+        send_task_notification(task)
+    else:
+        await sync_to_async(task.set_completed)(result)
+        send_task_notification(task)
+        log_task_completed(task, domain)
+        return result
+    return None
+
+
+def log_task_start(task, domain):
+    _logger.info(
+        "Starting task id: %s in domain: %s, task_type: %s, immediate: %s, deferred: %s",
+        task.pk,
+        domain.name,
+        task.name,
+        str(task.immediate),
+        str(task.deferred),
+    )
+
+
+def log_task_completed(task, domain):
+    execution_time = task.finished_at - task.started_at
+    execution_time_us = int(execution_time.total_seconds() * 1_000_000)  # μs
+    _logger.info(
+        "Task completed %s in domain:"
+        " %s, task_type: %s, immediate: %s, deferred: %s, execution_time: %s μs",
+        task.pk,
+        domain.name,
+        task.name,
+        str(task.immediate),
+        str(task.deferred),
+        execution_time_us,
+    )
+
+
+def log_task_failed(task, exc_type, exc, tb, domain):
+    _logger.info(
+        "Task[{task_type}] {task_pk} failed ({exc_type}: {exc}) in domain: {domain}".format(
+            task_type=task.name,
+            task_pk=task.pk,
+            exc_type=exc_type.__name__,
+            exc=exc,
+            domain=domain.name,
+        )
+    )
+    _logger.info("\n".join(traceback.format_list(traceback.extract_tb(tb))))
+
+
+def get_task_function(task, ensure_coroutine=False):
+    module_name, function_name = task.name.rsplit(".", 1)
+    module = importlib.import_module(module_name)
+    func = getattr(module, function_name)
+    args = task.enc_args or ()
+    kwargs = task.enc_kwargs or {}
+    immediate = task.immediate
+    is_coroutine_fn = asyncio.iscoroutinefunction(func)
+
+    if immediate and not is_coroutine_fn:
+        raise ValueError("Immediate tasks must be async functions.")
+
+    if ensure_coroutine:
+        if not is_coroutine_fn:
+            return sync_to_async(func)(*args, **kwargs)
+        coro = func(*args, **kwargs)
+        if immediate:
+            coro = asyncio.wait_for(coro, timeout=IMMEDIATE_TIMEOUT)
+        return coro
+    else:  # ensure normal function
+        if not is_coroutine_fn:
+            return lambda: func(*args, **kwargs)
+
+        async def task_wrapper():  # asyncio.wait_for + async_to_sync requires wrapping
+            coro = func(*args, **kwargs)
+            if immediate:
+                coro = asyncio.wait_for(coro, timeout=IMMEDIATE_TIMEOUT)
+            try:
+                return await coro
+            except asyncio.TimeoutError:
+                msg_template = "Immediate task %s timed out after %s seconds."
+                error_msg = msg_template % (task.pk, IMMEDIATE_TIMEOUT)
+                _logger.info(error_msg)
+                raise RuntimeError(error_msg)
+
+        return async_to_sync(task_wrapper)
 
 
 def dispatch(
@@ -194,18 +233,148 @@ def dispatch(
         ValueError: When `resources` is an unsupported type.
     """
 
-    # Can't run short tasks immediately if running from thread pool
-    immediate = immediate and not running_from_thread_pool()
+    execute_now = immediate and not called_from_content_app()
     assert deferred or immediate, "A task must be at least `deferred` or `immediate`."
+    send_wakeup_signal = True if not immediate else False
+    function_name = get_function_name(func)
+    versions = get_version(versions, function_name)
+    colliding_resources, resources = get_resources(exclusive_resources, shared_resources, immediate)
+    task_payload = get_task_payload(
+        function_name, task_group, args, kwargs, resources, versions, immediate, deferred
+    )
+    task = Task.objects.create(**task_payload)
+    task.refresh_from_db()  # The database will have assigned a timestamp for us.
+    if execute_now:
+        if are_resources_available(colliding_resources, task):
+            send_wakeup_signal = True if resources else False
+            task.unblock()
+            with using_workdir():
+                execute_task(task)
+        elif deferred:  # Resources are blocked and can be deferred
+            task.app_lock = None
+            task.save()
+        else:  # Can't be deferred
+            task.set_canceling()
+            task.set_canceled(TASK_STATES.CANCELED, "Resources temporarily unavailable.")
+    if send_wakeup_signal:
+        wakeup_worker(TASK_WAKEUP_UNBLOCK)
+    return task
+
+
+async def adispatch(
+    func,
+    args=None,
+    kwargs=None,
+    task_group=None,
+    exclusive_resources=None,
+    shared_resources=None,
+    immediate=False,
+    deferred=True,
+    versions=None,
+):
+    """Async version of dispatch."""
+    execute_now = immediate and not called_from_content_app()
+    assert deferred or immediate, "A task must be at least `deferred` or `immediate`."
+    function_name = get_function_name(func)
+    versions = get_version(versions, function_name)
+    colliding_resources, resources = get_resources(exclusive_resources, shared_resources, immediate)
+    send_wakeup_signal = False
+    task_payload = get_task_payload(
+        function_name, task_group, args, kwargs, resources, versions, immediate, deferred
+    )
+    task = await Task.objects.acreate(**task_payload)
+    await task.arefresh_from_db()  # The database will have assigned a timestamp for us.
+    if execute_now:
+        if await async_are_resources_available(colliding_resources, task):
+            send_wakeup_signal = True if resources else False
+            await task.aunblock()
+            with using_workdir():
+                await aexecute_task(task)
+        elif deferred:  # Resources are blocked and can be deferred
+            task.app_lock = None
+            await task.asave()
+        else:  # Can't be deferred
+            task.set_canceling()
+            task.set_canceled(TASK_STATES.CANCELED, "Resources temporarily unavailable.")
+    if send_wakeup_signal:
+        wakeup_worker(TASK_WAKEUP_UNBLOCK)
+    return task
+
+
+def get_task_payload(
+    function_name, task_group, args, kwargs, resources, versions, immediate, deferred
+):
+    payload = {
+        "state": TASK_STATES.WAITING,
+        "logging_cid": (get_guid()),
+        "task_group": task_group,
+        "name": function_name,
+        "enc_args": args,
+        "enc_kwargs": kwargs,
+        "parent_task": Task.current(),
+        "reserved_resources_record": resources,
+        "versions": versions,
+        "immediate": immediate,
+        "deferred": deferred,
+        "profile_options": x_task_diagnostics_var.get(None),
+        "app_lock": None if not immediate else AppStatus.objects.current(),  # Lazy evaluation...
+    }
+    return payload
+
+
+@contextmanager
+def using_workdir():
+    cur_dir = os.getcwd()
+    with tempfile.TemporaryDirectory(dir=settings.WORKING_DIRECTORY) as working_dir:
+        os.chdir(working_dir)
+        try:
+            yield
+        finally:
+            # Whether the task fails or not, we should always restore the workdir.
+            os.chdir(cur_dir)
+
 
+async def async_are_resources_available(colliding_resources, task: Task) -> bool:
+    prior_tasks = Task.objects.filter(
+        state__in=TASK_INCOMPLETE_STATES, pulp_created__lt=task.pulp_created
+    )
+    colliding_resources_taken = await prior_tasks.filter(
+        reserved_resources_record__overlap=colliding_resources
+    ).aexists()
+    return not colliding_resources or not colliding_resources_taken
+
+
+def are_resources_available(colliding_resources, task: Task) -> bool:
+    prior_tasks = Task.objects.filter(
+        state__in=TASK_INCOMPLETE_STATES, pulp_created__lt=task.pulp_created
+    )
+    colliding_resources_taken = prior_tasks.filter(
+        reserved_resources_record__overlap=colliding_resources
+    ).exists()
+    return not colliding_resources or not colliding_resources_taken
+
+
+def called_from_content_app() -> bool:
+    current_app = AppStatus.objects.current()
+    return current_app is not None and current_app.app_type == "content"
+
+
+def get_function_name(func):
     if callable(func):
         function_name = f"{func.__module__}.{func.__name__}"
     else:
         function_name = func
+    return function_name
+
 
+def get_version(versions, function_name):
     if versions is None:
         versions = MODULE_PLUGIN_VERSIONS[function_name.split(".", maxsplit=1)[0]]
+    return versions
+
 
+def get_resources(exclusive_resources, shared_resources, immediate):
+    domain_prn = get_prn(get_domain())
     if exclusive_resources is None:
         exclusive_resources = []
     else:
@@ -216,70 +385,19 @@ def dispatch(
         shared_resources = _validate_and_get_resources(shared_resources)
 
     # A task that is exclusive on a domain will block all tasks within that domain
-    domain_prn = get_prn(get_domain())
     if domain_prn not in exclusive_resources:
         shared_resources.append(domain_prn)
     resources = exclusive_resources + [f"shared:{resource}" for resource in shared_resources]
 
-    notify_workers = False
-    task = Task.objects.create(
-        state=TASK_STATES.WAITING,
-        logging_cid=(get_guid()),
-        task_group=task_group,
-        name=function_name,
-        enc_args=args,
-        enc_kwargs=kwargs,
-        parent_task=Task.current(),
-        reserved_resources_record=resources,
-        versions=versions,
-        immediate=immediate,
-        deferred=deferred,
-        profile_options=x_task_diagnostics_var.get(None),
-        app_lock=None if not immediate else AppStatus.objects.current(),  # Lazy evaluation...
-    )
-    task.refresh_from_db()  # The database will have assigned a timestamp for us.
+    # Compile a list of resources that must not be taken by other tasks.
+    colliding_resources = []
     if immediate:
-        prior_tasks = Task.objects.filter(
-            state__in=TASK_INCOMPLETE_STATES, pulp_created__lt=task.pulp_created
-        )
-        # Compile a list of resources that must not be taken by other tasks.
         colliding_resources = (
             shared_resources
             + exclusive_resources
             + [f"shared:{resource}" for resource in exclusive_resources]
         )
-        # Can we execute this task immediately?
-        if (
-            not colliding_resources
-            or not prior_tasks.filter(
-                reserved_resources_record__overlap=colliding_resources
-            ).exists()
-        ):
-            task.unblock()
-
-            cur_dir = os.getcwd()
-            with tempfile.TemporaryDirectory(dir=settings.WORKING_DIRECTORY) as working_dir:
-                os.chdir(working_dir)
-                try:
-                    execute_task(task)
-                finally:
-                    # Whether the task fails or not, we should always restore the workdir.
-                    os.chdir(cur_dir)
-
-            if resources:
-                notify_workers = True
-        elif deferred:
-            # Resources are blocked. Let the others handle it.
-            task.app_lock = None
-            task.save()
-        else:
-            task.set_canceling()
-            task.set_canceled(TASK_STATES.CANCELED, "Resources temporarily unavailable.")
-    else:
-        notify_workers = True
-    if notify_workers:
-        wakeup_worker(TASK_WAKEUP_UNBLOCK)
-    return task
+    return colliding_resources, resources
 
 
 def cancel_task(task_id):