pulpcore 3.76.1__py3-none-any.whl → 3.77.1__py3-none-any.whl

pulp_certguard/app/__init__.py

@@ -6,6 +6,6 @@ class PulpCertGuardPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_certguard.app"
     label = "certguard"
-    version = "3.76.1"
+    version = "3.77.1"
     python_package_name = "pulpcore"
     domain_compatible = True

pulp_file/app/__init__.py

@@ -8,6 +8,6 @@ class PulpFilePluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_file.app"
     label = "file"
-    version = "3.76.1"
+    version = "3.77.1"
     python_package_name = "pulpcore"
     domain_compatible = True

pulp_file/tests/functional/api/test_remote_settings.py

@@ -1,10 +1,89 @@
-import uuid
-
 import pytest
+import uuid
 
 from pulpcore.client.pulp_file import (
     RepositorySyncURL,
 )
+from pulpcore.client.pulp_file.exceptions import BadRequestException
+
+GOOD_CERT = """-----BEGIN CERTIFICATE-----
+MIICoDCCAYgCCQC2c2uY34HNlzANBgkqhkiG9w0BAQUFADASMRAwDgYDVQQDDAdn
+b3ZlZ2FuMB4XDTE5MDMxMzIxMDMzMFoXDTM4MDYxNjIxMDMzMFowEjEQMA4GA1UE
+AwwHZ292ZWdhbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANEatWsZ
+1iwGmTxD02dxMI4ci+Au4FzvmWLBWD07H5GGTVFwnqmNOKhP6DHs1EsMZevkUvaG
+CRxZlPYhjNFLZr2c2FnoDZ5nBXlSW6sodXURbMfyT187nDeBXVYFuh4T2eNCatnm
+t3vgdi+pWsF0LbOgpu7GJI2sh5K1imxyB77tJ7PFTDZCSohkK+A+0nDCnJqDUNXD
+5CK8iaBciCbnzp3nRKuM2EmgXno9Repy/HYxIgB7ZodPwDvYNjMGfvs0s9mJIKmc
+CKgkPXVO9y9gaRrrytICcPOs+YoU/PN4Ttg6wzxaWvJgw44vsR8wM/0i4HlXfBdl
+9br+cgn8jukDOgECAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAyNHV6NA+0GfUrvBq
+AHXHNnBE3nzMhGPhF/0B/dO4o0n6pgGZyzRxaUaoo6+5oQnBf/2NmDyLWdalFWX7
+D1WBaxkhK+FU922+qwQKhABlwMxGCnfZ8F+rlk4lNotm3fP4wHbnO1SGIDvvZFt/
+mpMgkhwL4lShUFv57YylXr+D2vSFcAryKiVGk1X3sHMXlFAMLHUm3d97fJnmb1qQ
+wC43BlJCBQF98wKtYNwTUG/9gblfk8lCB2DL1hwmPy3q9KbSDOdUK3HW6a75ZzCD
+6mXc/Y0bJcwweDsywbPBYP13hYUcpw4htcU6hg6DsoAjLNkSrlY+GGo7htx+L9HH
+IwtfRg==
+-----END CERTIFICATE-----
+"""
+
+GOOD_CERT_WITH_COMMENT = """saydas Intermédiaire CA
+-----BEGIN CERTIFICATE-----
+MIICoDCCAYgCCQC2c2uY34HNlzANBgkqhkiG9w0BAQUFADASMRAwDgYDVQQDDAdn
+b3ZlZ2FuMB4XDTE5MDMxMzIxMDMzMFoXDTM4MDYxNjIxMDMzMFowEjEQMA4GA1UE
+AwwHZ292ZWdhbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANEatWsZ
+1iwGmTxD02dxMI4ci+Au4FzvmWLBWD07H5GGTVFwnqmNOKhP6DHs1EsMZevkUvaG
+CRxZlPYhjNFLZr2c2FnoDZ5nBXlSW6sodXURbMfyT187nDeBXVYFuh4T2eNCatnm
+t3vgdi+pWsF0LbOgpu7GJI2sh5K1imxyB77tJ7PFTDZCSohkK+A+0nDCnJqDUNXD
+5CK8iaBciCbnzp3nRKuM2EmgXno9Repy/HYxIgB7ZodPwDvYNjMGfvs0s9mJIKmc
+CKgkPXVO9y9gaRrrytICcPOs+YoU/PN4Ttg6wzxaWvJgw44vsR8wM/0i4HlXfBdl
+9br+cgn8jukDOgECAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAyNHV6NA+0GfUrvBq
+AHXHNnBE3nzMhGPhF/0B/dO4o0n6pgGZyzRxaUaoo6+5oQnBf/2NmDyLWdalFWX7
+D1WBaxkhK+FU922+qwQKhABlwMxGCnfZ8F+rlk4lNotm3fP4wHbnO1SGIDvvZFt/
+mpMgkhwL4lShUFv57YylXr+D2vSFcAryKiVGk1X3sHMXlFAMLHUm3d97fJnmb1qQ
+wC43BlJCBQF98wKtYNwTUG/9gblfk8lCB2DL1hwmPy3q9KbSDOdUK3HW6a75ZzCD
+6mXc/Y0bJcwweDsywbPBYP13hYUcpw4htcU6hg6DsoAjLNkSrlY+GGo7htx+L9HH
+IwtfRg==
+-----END CERTIFICATE-----
+"""
+
+GOOD_CERT_TWO = """-----BEGIN CERTIFICATE-----
+MIICqjCCAZICAgtCMA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNVBAMMB2dvdmVnYW4w
+HhcNMTkwMzEzMjEwMzMwWhcNMjkwMzEwMjEwMzMwWjAjMRAwDgYDVQQDDAdnb3Zl
+Z2FuMQ8wDQYDVQQKDAZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCxJWx5t25jY4womWtKxGqv2LHg9YnU0b2VCECLhu5JjoAzFPja5VHB0Maz
+G8m5c0+N2ubrPcBC+KdoGMd2MqrrGyzKOiwbVDW0YOgnFqh58p796iKtVboWx41y
+Gzn289PzYccxH6mhhPmRVD25KyV1TenqvGIHJTepF7mgIemGDTv+j7+mYPT/3r6I
+pnwTkEVPr+Q4iW0l3fNESlFFRt2b7yhz9f0E4SMhmIRnSIGOLO1zE02IJ1hTuGkx
+/MZ1AqQdVSdm4jenTIMp91R1kYylI66yMcpU6w6x4j8qvJ8nBZ4r4DqOHcOobyHp
+qlaJjv/K5SGJxV2k0EFk7b483lbrAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBALYh
+SHLGJCxVL8ePFLs294fhTq4pTQsvHm8q3SyJD9DaB+HKTceCFErNv18Dsl/QwBis
+WPHWKpDN0EUcuuE/8oUaGjjzByJ8bPafMicFCHSSefcJw+IOOqKBkWDT+4YGkvfs
+RpwxSLqLOhEt7aSkiPcMvD20v8cvj0O36c5G3Vv0E8WmPWOEqjyPFoU9X1vACr+h
+DdIKvxFbvRU9ObektFxOYHuvP010IBv2dGyw3G5W5fh9A5OSXHAShWSwkRU36oft
+ugB47fIIlb7zLm4GBmxGG0yBwAf4otBlUXVNqNx15bbUuVgKbGMFfItQgEo9AQcz
+gGsetwDOs/NgZ95oH40=
+-----END CERTIFICATE-----
+"""
+
+BAD_CERT = """-----BEGIN CERTIFICATE-----\nBOGUS==\n-----END CERTIFICATE-----
+"""
+
+NO_CERT = """
+MIICoDCCAYgCCQC2c2uY34HNlzANBgkqhkiG9w0BAQUFADASMRAwDgYDVQQDDAdn
+b3ZlZ2FuMB4XDTE5MDMxMzIxMDMzMFoXDTM4MDYxNjIxMDMzMFowEjEQMA4GA1UE
+AwwHZ292ZWdhbjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANEatWsZ
+1iwGmTxD02dxMI4ci+Au4FzvmWLBWD07H5GGTVFwnqmNOKhP6DHs1EsMZevkUvaG
+CRxZlPYhjNFLZr2c2FnoDZ5nBXlSW6sodXURbMfyT187nDeBXVYFuh4T2eNCatnm
+t3vgdi+pWsF0LbOgpu7GJI2sh5K1imxyB77tJ7PFTDZCSohkK+A+0nDCnJqDUNXD
+5CK8iaBciCbnzp3nRKuM2EmgXno9Repy/HYxIgB7ZodPwDvYNjMGfvs0s9mJIKmc
+CKgkPXVO9y9gaRrrytICcPOs+YoU/PN4Ttg6wzxaWvJgw44vsR8wM/0i4HlXfBdl
+9br+cgn8jukDOgECAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAyNHV6NA+0GfUrvBq
+AHXHNnBE3nzMhGPhF/0B/dO4o0n6pgGZyzRxaUaoo6+5oQnBf/2NmDyLWdalFWX7
+D1WBaxkhK+FU922+qwQKhABlwMxGCnfZ8F+rlk4lNotm3fP4wHbnO1SGIDvvZFt/
+mpMgkhwL4lShUFv57YylXr+D2vSFcAryKiVGk1X3sHMXlFAMLHUm3d97fJnmb1qQ
+wC43BlJCBQF98wKtYNwTUG/9gblfk8lCB2DL1hwmPy3q9KbSDOdUK3HW6a75ZzCD
+6mXc/Y0bJcwweDsywbPBYP13hYUcpw4htcU6hg6DsoAjLNkSrlY+GGo7htx+L9HH
+IwtfRg==
+"""
 
 
 def _run_basic_sync_and_assert(file_bindings, remote, file_repo, monitor_task):
@@ -211,3 +290,55 @@ def test_header_for_sync(
     assert requests_record[0].path == "/basic/PULP_MANIFEST"
     assert header_name in requests_record[0].headers
     assert header_value == requests_record[0].headers[header_name]
+
+
+@pytest.mark.parallel
+def test_certificate_clean(file_remote_factory):
+    # Check that a good cert validates
+    a_remote = file_remote_factory(url="http://example.com/", ca_cert=GOOD_CERT)
+    assert a_remote.ca_cert == GOOD_CERT
+    a_remote = file_remote_factory(url="http://example.com/", client_cert=GOOD_CERT)
+    assert a_remote.client_cert == GOOD_CERT
+
+    # Check that a good-cert-with-comments validates and strips the comments
+    a_remote = file_remote_factory(url="http://example.com/", ca_cert=GOOD_CERT_WITH_COMMENT)
+    assert a_remote.ca_cert == GOOD_CERT
+    a_remote = file_remote_factory(url="http://example.com/", client_cert=GOOD_CERT_WITH_COMMENT)
+    assert a_remote.client_cert == GOOD_CERT
+
+    # Check that a bad-cert gets rejected
+    with pytest.raises(BadRequestException):
+        a_remote = file_remote_factory(url="http://example.com/", ca_cert=BAD_CERT)
+    with pytest.raises(BadRequestException):
+        a_remote = file_remote_factory(url="http://example.com/", client_cert=BAD_CERT)
+
+    # Check that a no-cert string returns the expected error
+    with pytest.raises(BadRequestException):
+        a_remote = file_remote_factory(url="http://example.com/", ca_cert=NO_CERT)
+    with pytest.raises(BadRequestException):
+        a_remote = file_remote_factory(url="http://example.com/", client_cert=NO_CERT)
+
+
+@pytest.mark.parallel
+def test_multi_certificate_clean(file_remote_factory):
+    multi_cert = GOOD_CERT + GOOD_CERT_TWO
+
+    # Check that a good multi-cert PEM validates
+    a_remote = file_remote_factory(url="http://example.com/", ca_cert=multi_cert)
+    assert a_remote.ca_cert == multi_cert
+    a_remote = file_remote_factory(url="http://example.com/", client_cert=multi_cert)
+    assert a_remote.client_cert == multi_cert
+
+    # Check that a good-cert-with-comments validates and strips the comments
+    multi_cert_with_comment = GOOD_CERT_WITH_COMMENT + GOOD_CERT_TWO
+    a_remote = file_remote_factory(url="http://example.com/", ca_cert=multi_cert_with_comment)
+    assert a_remote.ca_cert == multi_cert
+    a_remote = file_remote_factory(url="http://example.com/", client_cert=multi_cert_with_comment)
+    assert a_remote.client_cert == multi_cert
+
+    # Check that multi-with-bad is rejected
+    multi_bad = GOOD_CERT + BAD_CERT
+    with pytest.raises(BadRequestException):
+        a_remote = file_remote_factory(url="http://example.com/", ca_cert=multi_bad)
+    with pytest.raises(BadRequestException):
+        a_remote = file_remote_factory(url="http://example.com/", client_cert=multi_bad)

pulpcore/app/apps.py

@@ -247,7 +247,7 @@ class PulpAppConfig(PulpPluginAppConfig):
     label = "core"
 
     # The version of this app
-    version = "3.76.1"
+    version = "3.77.1"
 
     # The python package name providing this app
     python_package_name = "pulpcore"

pulpcore/app/replica.py

@@ -8,7 +8,7 @@ from pulp_glue.common.context import PulpContext
 from pulpcore.app.models import UpstreamPulp
 from pulpcore.tasking.tasks import dispatch
 from pulpcore.app.tasks.base import (
-    general_update,
+    ageneral_update,
     general_create,
     general_multi_delete,
 )
@@ -133,7 +133,7 @@ class Replicator:
             needs_update = self.needs_update(remote_fields_dict, remote)
             if needs_update:
                 dispatch(
-                    general_update,
+                    ageneral_update,
                     task_group=self.task_group,
                     shared_resources=[self.server],
                     exclusive_resources=[remote],
@@ -162,7 +162,7 @@ class Replicator:
             needs_update = self.needs_update(repo_fields_dict, repository)
             if needs_update:
                 dispatch(
-                    general_update,
+                    ageneral_update,
                     task_group=self.task_group,
                     shared_resources=[self.server],
                     exclusive_resources=[repository],
@@ -197,7 +197,7 @@ class Replicator:
             if needs_update:
                 # Update the distribution
                 dispatch(
-                    general_update,
+                    ageneral_update,
                     task_group=self.task_group,
                     shared_resources=[repository, self.server],
                     exclusive_resources=self.distros_uris,

pulpcore/app/serializers/publication.py

@@ -407,14 +407,34 @@ def validate_repo_and_remote(repository, field, **kwargs):
 
     detail_repo = repository.cast()
     if detail_repo.remote and type(detail_repo.remote.cast()) not in detail_repo.REMOTE_TYPES:
-        msg = (
-            f"Type for Remote '{get_prn(repository.remote)}' "
-            f"does not match Repository '{get_prn(repository)}'"
-        )
-        if repository_version := kwargs.pop("repository_version", None):
-            msg += f" from RepositoryVersion '{get_prn(repository_version)}'"
-        if publication := kwargs.pop("publication", None):
-            msg += f" from Publication '{get_prn(publication)}'"
-        msg += "."
+        repository_version = kwargs.pop("repository_version", None)
+        publication = kwargs.pop("publication", None)
+
+        if repository_version and publication:
+            msg_template = _(
+                "Type for Remote '{remote}' does not match Repository '{repository}' "
+                "from RepositoryVersion '{repository_version}' from Publication '{publication}'."
+            )
+        elif repository_version:
+            msg_template = _(
+                "Type for Remote '{remote}' does not match Repository '{repository}' "
+                "from RepositoryVersion '{repository_version}'."
+            )
+        elif publication:
+            msg_template = _(
+                "Type for Remote '{remote}' does not match Repository '{repository}' "
+                "from Publication '{publication}'."
+            )
+        else:
+            msg_template = _("Type for Remote '{remote}' does not match Repository '{repository}'.")
 
-        raise serializers.ValidationError({field: _(msg)})
+        raise serializers.ValidationError(
+            {
+                field: msg_template.format(
+                    remote=get_prn(repository.remote),
+                    repository=get_prn(repository),
+                    repository_version=get_prn(repository_version) if repository_version else "",
+                    publication=get_prn(publication) if publication else "",
+                )
+            }
+        )

pulpcore/app/serializers/repository.py

@@ -1,4 +1,5 @@
 import os
+from cryptography.x509 import load_pem_x509_certificate
 from gettext import gettext as _
 from urllib.parse import urlparse
 
@@ -72,6 +73,46 @@ class RepositorySerializer(ModelSerializer):
         )
 
 
+def validate_certificate(which_cert, value):
+    """
+    Validate and return *just* the certs and not any commentary that came along with them.
+
+    Args:
+        which_cert: The attribute-name whose cert we're validating (only used for error-message).
+        value: The string being proposed as a certificate-containing PEM.
+
+    Raises:
+        ValidationError: When the provided value has no or an invalid certificate.
+
+    Returns:
+        The pem-string with *just* the validated BEGIN/END CERTIFICATE segments.
+    """
+    if value:
+        try:
+            # Find any/all CERTIFICATE entries in the proposed PEM and let crypto validate them.
+            # NOTE: crypto/39 includes load_certificates(), which will let us remove this whole
+            # loop. But we want to fix the current problem on older supported branches that
+            # allow 38, so we do it ourselves for now
+            certs = list()
+            a_cert = ""
+            for line in value.split("\n"):
+                if "-----BEGIN CERTIFICATE-----" in line or a_cert:
+                    a_cert += line + "\n"
+                if "-----END CERTIFICATE-----" in line:
+                    load_pem_x509_certificate(bytes(a_cert, "ASCII"))
+                    certs.append(a_cert.strip())
+                    a_cert = ""
+            if not certs:
+                raise serializers.ValidationError(
+                    "No {} specified in string {}".format(which_cert, value)
+                )
+            return "\n".join(certs) + "\n"
+        except ValueError as e:
+            raise serializers.ValidationError(
+                "Invalid {} specified, error '{}'".format(which_cert, e.args)
+            )
+
+
 class RemoteSerializer(ModelSerializer, HiddenFieldsMixin):
     """
     Every remote defined by a plugin should have a Remote serializer that inherits from this
@@ -271,6 +312,12 @@ class RemoteSerializer(ModelSerializer, HiddenFieldsMixin):
             raise serializers.ValidationError(_("proxy_url must not contain credentials"))
         return value
 
+    def validate_ca_cert(self, value):
+        return validate_certificate("ca_cert", value)
+
+    def validate_client_cert(self, value):
+        return validate_certificate("client_cert", value)
+
     def validate(self, data):
         """
         Check, that proxy credentials are only provided completely and if a proxy is configured.
@@ -365,9 +412,8 @@ class RepositorySyncURLSerializer(ValidateFieldsMixin, serializers.Serializer):
         if repository and type(remote.cast()) not in repository.cast().REMOTE_TYPES:
             raise serializers.ValidationError(
                 {
-                    "remote": _(
-                        f"Type for Remote '{get_prn(remote)}' "
-                        f"does not match Repository '{get_prn(repository)}'."
+                    "remote": _("Type for Remote '{}' does not match Repository '{}'.").format(
+                        get_prn(remote), get_prn(repository)
                     )
                 }
             )

pulpcore/app/settings.py

@@ -382,7 +382,19 @@ ALLOWED_CONTENT_CHECKSUMS = ["sha224", "sha256", "sha384", "sha512"]
 
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
-TASK_DIAGNOSTICS = False
+# Possible diagnostics:
+# * "memory"
+#    Logs the task process RSS every couple of seconds
+# * "pyinstrument"
+#    Dumps an HTML profile report produced by pyinstrument, showing time spent in various
+#    callstacks. This adds ~10% overhead to the task process and consumes extra memory.
+#    Tweaking code might be warranted for some advanced settings.
+# * "memray" - Dumps a report produced by memray which logs how much memory was allocated by which
+#    lines and functions, at the time of peak RSS of the task process. This adds significant
+#    runtime overhead to the task process, 20-40%. Tweaking code might be warranted for
+#    some advanced settings.
+# NOTE: "memray" and "pyinstrument" require additional packages to be installed on the system.
+TASK_DIAGNOSTICS = []  # ["memory", "pyinstrument", "memray"]
 
 ANALYTICS = True
 

pulpcore/app/tasks/__init__.py

@@ -1,11 +1,11 @@
 from pulpcore.app.tasks import base, repository, upload
 
 from .base import (
+    ageneral_update,
     general_create,
     general_create_from_temp_file,
     general_delete,
     general_multi_delete,
-    general_update,
 )
 
 from .export import fs_publication_export, fs_repo_version_export

pulpcore/app/tasks/base.py

@@ -2,8 +2,11 @@ from django.db import transaction
 
 from pulpcore.app.apps import get_plugin_config
 from pulpcore.app.models import CreatedResource
+from pulpcore.app.loggers import deprecation_logger
 from pulpcore.plugin.models import MasterModel
 
+from asgiref.sync import sync_to_async
+
 
 def general_create_from_temp_file(app_label, serializer_name, temp_file_pk, *args, **kwargs):
     """
@@ -63,6 +66,10 @@ def general_update(instance_id, app_label, serializer_name, *args, **kwargs):
             due to validation error. This theoretically should never occur since validation is
             performed before the task is dispatched.
     """
+    deprecation_logger.warning(
+        "`pulpcore.app.tasks.base.general_update` is deprecated and will be removed in Pulp 4. "
+        "Use `pulpcore.app.tasks.base.ageneral_update` instead."
+    )
     data = kwargs.pop("data", None)
     partial = kwargs.pop("partial", False)
     serializer_class = get_plugin_config(app_label).named_serializers[serializer_name]
@@ -85,6 +92,10 @@ def general_delete(instance_id, app_label, serializer_name):
         app_label (str): the Django app label of the plugin that provides the model
         serializer_name (str): name of the serializer class for the model
     """
+    deprecation_logger.warning(
+        "`pulpcore.app.tasks.base.general_delete` is deprecated and will be removed in Pulp 4. "
+        "Use `pulpcore.app.tasks.base.ageneral_delete` instead."
+    )
     serializer_class = get_plugin_config(app_label).named_serializers[serializer_name]
     instance = serializer_class.Meta.model.objects.get(pk=instance_id)
     if isinstance(instance, MasterModel):
@@ -111,3 +122,29 @@ def general_multi_delete(instance_ids):
     with transaction.atomic():
         for instance in instances:
             instance.delete()
+
+
+async def ageneral_update(instance_id, app_label, serializer_name, *args, **kwargs):
+    """
+    Async version of [pulpcore.app.tasks.base.general_update][].
+    """
+    data = kwargs.pop("data", None)
+    partial = kwargs.pop("partial", False)
+    serializer_class = get_plugin_config(app_label).named_serializers[serializer_name]
+    instance = await serializer_class.Meta.model.objects.aget(pk=instance_id)
+    if isinstance(instance, MasterModel):
+        instance = await instance.acast()
+    serializer = serializer_class(instance, data=data, partial=partial)
+    await sync_to_async(serializer.is_valid)(raise_exception=True)
+    await sync_to_async(serializer.save)()
+
+
+async def ageneral_delete(instance_id, app_label, serializer_name):
+    """
+    Async version of [pulpcore.app.tasks.base.general_delete][].
+    """
+    serializer_class = get_plugin_config(app_label).named_serializers[serializer_name]
+    instance = await serializer_class.Meta.model.objects.aget(pk=instance_id)
+    if isinstance(instance, MasterModel):
+        instance = await instance.acast()
+    await instance.adelete()

pulpcore/app/tasks/test.py

@@ -1,4 +1,6 @@
+import asyncio
 import backoff
+import time
 from pulpcore.app.models import TaskGroup
 from pulpcore.tasking.tasks import dispatch
 
@@ -9,17 +11,18 @@ def dummy_task():
 
 
 def sleep(interval):
-    from time import sleep
+    time.sleep(interval)
 
-    sleep(interval)
+
+async def asleep(interval):
+    """Async function that sleeps."""
+    await asyncio.sleep(interval)
 
 
 @backoff.on_exception(backoff.expo, BaseException)
 def gooey_task(interval):
     """A sleep task that tries to avoid being killed by ignoring all exceptions."""
-    from time import sleep
-
-    sleep(interval)
+    time.sleep(interval)
 
 
 def dummy_group_task(inbetween=3, intervals=None):
@@ -28,5 +31,5 @@ def dummy_group_task(inbetween=3, intervals=None):
     task_group = TaskGroup.current()
     for interval in intervals:
         dispatch(sleep, args=(interval,), task_group=task_group)
-        sleep(inbetween)
+        time.sleep(inbetween)
     task_group.finish()

pulpcore/app/viewsets/base.py

@@ -492,7 +492,7 @@ class AsyncUpdateMixin(AsyncReservedObjectMixin):
         serializer.is_valid(raise_exception=True)
         app_label = instance._meta.app_label
         task = dispatch(
-            tasks.base.general_update,
+            tasks.base.ageneral_update,
             exclusive_resources=self.async_reserved_resources(instance),
             args=(pk, app_label, serializer.__class__.__name__),
             kwargs={"data": request.data, "partial": partial},
@@ -528,7 +528,7 @@ class AsyncRemoveMixin(AsyncReservedObjectMixin):
         serializer = self.get_serializer(instance)
         app_label = instance._meta.app_label
         task = dispatch(
-            tasks.base.general_delete,
+            tasks.base.ageneral_delete,
             exclusive_resources=self.async_reserved_resources(instance),
             args=(pk, app_label, serializer.__class__.__name__),
             immediate=self.ALLOW_NON_BLOCKING_DELETE,

pulpcore/cache/cache.py

@@ -8,7 +8,7 @@ from django.http import HttpResponseRedirect, HttpResponse, FileResponse as ApiF
 
 from rest_framework.request import Request as ApiRequest
 
-from aiohttp.web import FileResponse, Response, HTTPSuccessful, Request
+from aiohttp.web import FileResponse, Response, HTTPSuccessful, Request, StreamResponse
 from aiohttp.web_exceptions import HTTPFound
 
 from redis import ConnectionError
@@ -401,6 +401,11 @@ class AsyncContentCache(AsyncCache):
         except (HTTPSuccessful, HTTPFound) as e:
             response = e
 
+        original_response = response
+        if isinstance(response, StreamResponse):
+            if hasattr(response, "future_response"):
+                response = response.future_response
+
         entry = {"headers": dict(response.headers), "status": response.status}
         if expires is not None:
             # Redis TTL is not sufficient: https://github.com/pulp/pulpcore/issues/4845
@@ -427,12 +432,12 @@ class AsyncContentCache(AsyncCache):
             entry["location"] = str(response.location)
             entry["type"] = "Redirect"
         else:
-            # We don't cache StreamResponses or errors
+            # We don't cache errors
             return response
 
         # TODO look into smaller format, maybe some compression on the text
         await self.set(key, json.dumps(entry), expires, base_key=base_key)
-        return response
+        return original_response
 
     def make_key(self, request):
         """Makes the key based off the request"""

pulpcore/constants.py

@@ -45,6 +45,8 @@ TASK_FINAL_STATES = (
 #: Tasks in an incomplete state have not finished their work yet.
 TASK_INCOMPLETE_STATES = (TASK_STATES.WAITING, TASK_STATES.RUNNING, TASK_STATES.CANCELING)
 
+#: Timeout for immediate tasks in seconds
+IMMEDIATE_TIMEOUT = 5
 
 SYNC_MODES = SimpleNamespace(ADDITIVE="additive", MIRROR="mirror")
 SYNC_CHOICES = (

pulpcore/content/handler.py

@@ -1093,26 +1093,8 @@ class Handler:
             ret.update({ca.relative_path: ca for ca in cas})
         return ret
 
-    async def _serve_content_artifact(self, content_artifact, headers, request):
-        """
-        Handle response for a Content Artifact with the file present.
-
-        Depending on where the file storage (e.g. filesystem, S3, etc) this could be responding with
-        the file (filesystem) or a redirect (S3).
-
-        Args:
-            content_artifact (pulpcore.app.models.ContentArtifact) The Content Artifact to
-                respond with.
-            headers (dict): A dictionary of response headers.
-            request(aiohttp.web.Request) The request to prepare a response for.
-
-        Raises:
-            [aiohttp.web_exceptions.HTTPFound][]: When we need to redirect to the file
-            NotImplementedError: If file is stored in a file storage we can't handle
-
-        Returns:
-            The [aiohttp.web.FileResponse][] for the file.
-        """
+    def _build_response_from_content_artifact(self, content_artifact, headers, request):
+        """Helper method to build the correct response to serve a ContentArtifact."""
 
         def _set_params_from_headers(hdrs, storage_domain):
             # Map standard-response-headers to storage-object-specific keys
@@ -1137,7 +1119,47 @@ class Handler:
         artifact_name = artifact_file.name
         domain = get_domain()
         storage = domain.get_storage()
+        headers["X-PULP-ARTIFACT-SIZE"] = str(artifact_file.size)
+
+        if domain.storage_class == "pulpcore.app.models.storage.FileSystem":
+            path = storage.path(artifact_name)
+            if not os.path.exists(path):
+                raise Exception(_("Expected path '{}' is not found").format(path))
+            return FileResponse(path, headers=headers)
+        elif not domain.redirect_to_object_storage:
+            return ArtifactResponse(content_artifact.artifact, headers=headers)
+        elif domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage":
+            return HTTPFound(_build_url(http_method=request.method), headers=headers)
+        elif domain.storage_class in (
+            "storages.backends.azure_storage.AzureStorage",
+            "storages.backends.gcloud.GoogleCloudStorage",
+        ):
+            return HTTPFound(_build_url(), headers=headers)
+        else:
+            raise NotImplementedError()
 
+    async def _serve_content_artifact(self, content_artifact, headers, request):
+        """
+        Handle response for a Content Artifact with the file present.
+
+        Depending on where the file storage (e.g. filesystem, S3, etc) this could be responding with
+        the file (filesystem) or a redirect (S3).
+
+        Args:
+            content_artifact (pulpcore.app.models.ContentArtifact) The Content Artifact to
+                respond with.
+            headers (dict): A dictionary of response headers.
+            request(aiohttp.web.Request) The request to prepare a response for.
+
+        Raises:
+            [aiohttp.web_exceptions.HTTPFound][]: When we need to redirect to the file
+            NotImplementedError: If file is stored in a file storage we can't handle
+            HTTPRequestRangeNotSatisfiable: If the request is for a range that is not
+                satisfiable.
+        Returns:
+            The [aiohttp.web.FileResponse][] for the file.
+        """
+        artifact_file = content_artifact.artifact.file
         content_length = artifact_file.size
 
         try:
@@ -1152,25 +1174,13 @@ class Handler:
             size = artifact_file.size or "*"
             raise HTTPRequestRangeNotSatisfiable(headers={"Content-Range": f"bytes */{size}"})
 
-        headers["X-PULP-ARTIFACT-SIZE"] = str(content_length)
         artifacts_size_counter.add(content_length)
 
-        if domain.storage_class == "pulpcore.app.models.storage.FileSystem":
-            path = storage.path(artifact_name)
-            if not os.path.exists(path):
-                raise Exception(_("Expected path '{}' is not found").format(path))
-            return FileResponse(path, headers=headers)
-        elif not domain.redirect_to_object_storage:
-            return ArtifactResponse(content_artifact.artifact, headers=headers)
-        elif domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage":
-            raise HTTPFound(_build_url(http_method=request.method), headers=headers)
-        elif domain.storage_class in (
-            "storages.backends.azure_storage.AzureStorage",
-            "storages.backends.gcloud.GoogleCloudStorage",
-        ):
-            raise HTTPFound(_build_url(), headers=headers)
+        response = self._build_response_from_content_artifact(content_artifact, headers, request)
+        if isinstance(response, HTTPFound):
+            raise response
         else:
-            raise NotImplementedError()
+            return response
 
     async def _stream_remote_artifact(
         self, request, response, remote_artifact, save_artifact=True, repository=None
@@ -1214,6 +1224,7 @@ class Handler:
             size = remote_artifact.size or "*"
             raise HTTPRequestRangeNotSatisfiable(headers={"Content-Range": f"bytes */{size}"})
 
+        original_headers = response.headers.copy()
         actual_content_length = None
 
         if range_start or range_stop:
@@ -1323,9 +1334,14 @@ class Handler:
             content_artifacts = await asyncio.shield(
                 sync_to_async(self._save_artifact)(download_result, remote_artifact, request)
             )
+            ca = content_artifacts[remote_artifact.content_artifact.relative_path]
+            # If cache is enabled, add the future response to our stream response
+            if settings.CACHE_ENABLED:
+                response.future_response = self._build_response_from_content_artifact(
+                    ca, original_headers, request
+                )
             # Try to add content to repository if present & supported
             if repository and repository.PULL_THROUGH_SUPPORTED:
-                ca = content_artifacts[remote_artifact.content_artifact.relative_path]
                 await sync_to_async(repository.pull_through_add_content)(ca)
         await response.write_eof()