pulpcore 3.76.1__py3-none-any.whl → 3.77.1__py3-none-any.whl

pulpcore/plugin/tasking.py

@@ -2,13 +2,13 @@
 from pulpcore.tasking.tasks import dispatch
 
 from pulpcore.app.tasks import (
+    ageneral_update,
     fs_publication_export,
     fs_repo_version_export,
     general_create,
     general_create_from_temp_file,
     general_delete,
     general_multi_delete,
-    general_update,
     orphan_cleanup,
     reclaim_space,
 )
@@ -16,6 +16,7 @@ from pulpcore.app.tasks.repository import add_and_remove
 
 
 __all__ = [
+    "ageneral_update",
     "dispatch",
     "fs_publication_export",
     "fs_repo_version_export",
@@ -23,7 +24,6 @@ __all__ = [
     "general_create_from_temp_file",
     "general_delete",
     "general_multi_delete",
-    "general_update",
     "orphan_cleanup",
     "reclaim_space",
     "add_and_remove",

pulpcore/pytest_plugin.py

@@ -9,6 +9,7 @@ import shutil
 import socket
 import ssl
 import subprocess
+import sys
 import threading
 import uuid
 
@@ -1017,6 +1018,7 @@ def dispatch_task(pulpcore_bindings):
 
         assert process.returncode == 0
         task_href = process.stdout.decode().strip()
+        print(process.stderr.decode(), file=sys.stderr)
         return task_href
 
     return _dispatch_task

pulpcore/tasking/_util.py

@@ -71,8 +71,8 @@ def write_memory_usage(stop_event, path):
             current_mb_in_use = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
             file.write(f"{seconds}\t{current_mb_in_use:.2f}\n")
             file.flush()
-            time.sleep(5)
-            seconds += 5
+            time.sleep(2)
+            seconds += 2
 
 
 def child_signal_handler(sig, frame):
@@ -135,15 +135,21 @@ def perform_task(task_pk, task_working_dir_rel_path):
 
 def _execute_task_and_profile(task):
     with tempfile.TemporaryDirectory(dir=settings.WORKING_DIRECTORY) as temp_dir:
-        pyinstrument_func = _pyinstrument_diagnostic_decorator(temp_dir, execute_task)
-        memory_func = _memory_diagnostic_decorator(temp_dir, pyinstrument_func)
+        _execute_task = execute_task
 
-        memory_func(task)
+        if settings.TASK_DIAGNOSTICS is True or "memory" in settings.TASK_DIAGNOSTICS:
+            _execute_task = _memory_diagnostic_decorator(temp_dir, _execute_task)
+        if settings.TASK_DIAGNOSTICS is True or "pyinstrument" in settings.TASK_DIAGNOSTICS:
+            _execute_task = _pyinstrument_diagnostic_decorator(temp_dir, _execute_task)
+        if settings.TASK_DIAGNOSTICS is True or "memray" in settings.TASK_DIAGNOSTICS:
+            _execute_task = _memray_diagnostic_decorator(temp_dir, _execute_task)
+
+        _execute_task(task)
 
 
 def _memory_diagnostic_decorator(temp_dir, func):
     def __memory_diagnostic_decorator(task):
-        mem_diagnostics_file_path = os.path.join(temp_dir, "memory.datum")
+        mem_diagnostics_file_path = os.path.join(temp_dir, "memory_profile.datum")
         # It would be better to have this recording happen in the parent process instead of here
         # https://github.com/pulp/pulpcore/issues/2337
         stop_event = threading.Event()
@@ -158,10 +164,10 @@ def _memory_diagnostic_decorator(temp_dir, func):
         artifact = Artifact.init_and_validate(mem_diagnostics_file_path)
         with suppress(IntegrityError):
             artifact.save()
-
-        ProfileArtifact.objects.get_or_create(artifact=artifact, name="memory_profile", task=task)
-
-        _logger.info("Created memory diagnostic data.")
+            ProfileArtifact.objects.get_or_create(
+                artifact=artifact, name="memory_profile", task=task
+            )
+            _logger.info("Created memory diagnostic data.")
 
     return __memory_diagnostic_decorator
 
@@ -171,7 +177,7 @@ def _pyinstrument_diagnostic_decorator(temp_dir, func):
         if importlib.util.find_spec("pyinstrument") is not None:
             from pyinstrument import Profiler
 
-            with Profiler() as profiler:
+            with Profiler(interval=0.002) as profiler:
                 func(task)
 
             profile_file_path = os.path.join(temp_dir, "pyinstrument.html")
@@ -182,16 +188,40 @@ def _pyinstrument_diagnostic_decorator(temp_dir, func):
             artifact = Artifact.init_and_validate(str(profile_file_path))
             with suppress(IntegrityError):
                 artifact.save()
+                ProfileArtifact.objects.get_or_create(
+                    artifact=artifact, name="pyinstrument_profile", task=task
+                )
+                _logger.info("Created pyinstrument profile data.")
+        else:
+            func(task)
 
-            ProfileArtifact.objects.get_or_create(
-                artifact=artifact, name="pyinstrument_data", task=task
-            )
+    return __pyinstrument_diagnostic_decorator
+
+
+def _memray_diagnostic_decorator(temp_dir, func):
+    def __memray_diagnostic_decorator(task):
+        if importlib.util.find_spec("memray") is not None:
+            import memray
+
+            profile_file_path = os.path.join(temp_dir, "memray_profile.bin")
+            with memray.Tracker(
+                profile_file_path,
+                native_traces=False,
+                file_format=memray.FileFormat.AGGREGATED_ALLOCATIONS,
+            ):
+                func(task)
 
-            _logger.info("Created pyinstrument profile data.")
+            artifact = Artifact.init_and_validate(str(profile_file_path))
+            with suppress(IntegrityError):
+                artifact.save()
+                ProfileArtifact.objects.get_or_create(
+                    artifact=artifact, name="memray_profile", task=task
+                )
+                _logger.info("Created memray memory profile data.")
         else:
             func(task)
 
-    return __pyinstrument_diagnostic_decorator
+    return __memray_diagnostic_decorator
 
 
 def dispatch_scheduled_tasks():

pulpcore/tasking/tasks.py

@@ -7,6 +7,8 @@ import os
 import sys
 import traceback
 import tempfile
+import threading
+from asgiref.sync import sync_to_async
 from datetime import timedelta
 from gettext import gettext as _
 
@@ -16,12 +18,13 @@ from django.db.models import Model, Max
 from django_guid import get_guid
 from pulpcore.app.apps import MODULE_PLUGIN_VERSIONS
 from pulpcore.app.models import Task, TaskGroup
-from pulpcore.app.util import current_task, get_domain, get_prn
+from pulpcore.app.util import current_task, get_domain, get_prn, deprecation_logger
 from pulpcore.constants import (
     TASK_FINAL_STATES,
     TASK_INCOMPLETE_STATES,
     TASK_STATES,
     TASK_DISPATCH_LOCK,
+    IMMEDIATE_TIMEOUT,
 )
 from pulpcore.tasking.kafka import send_task_notification
 
@@ -75,11 +78,38 @@ def _execute_task(task):
         func = getattr(module, function_name)
         args = task.enc_args or ()
         kwargs = task.enc_kwargs or {}
-        result = func(*args, **kwargs)
-        if asyncio.iscoroutine(result):
+        immediate = task.immediate
+        is_coroutine_fn = asyncio.iscoroutinefunction(func)
+
+        if not is_coroutine_fn:
+            if immediate:
+                deprecation_logger.warning(
+                    "Immediate tasks must be coroutine functions. "
+                    "Support for non-coroutine immediate tasks will be dropped "
+                    "in pulpcore 3.85."
+                )
+                func = sync_to_async(func)
+                is_coroutine_fn = True
+            else:
+                func(*args, **kwargs)
+
+        if is_coroutine_fn:
             _logger.debug("Task is coroutine %s", task.pk)
+            coro = func(*args, **kwargs)
+            if immediate:
+                coro = asyncio.wait_for(coro, timeout=IMMEDIATE_TIMEOUT)
             loop = asyncio.get_event_loop()
-            loop.run_until_complete(result)
+            try:
+                loop.run_until_complete(coro)
+            except asyncio.TimeoutError:
+                _logger.info(
+                    "Immediate task %s timed out after %s seconds.", task.pk, IMMEDIATE_TIMEOUT
+                )
+                raise RuntimeError(
+                    "Immediate task timed out after {timeout} seconds.".format(
+                        timeout=IMMEDIATE_TIMEOUT,
+                    )
+                )
 
     except Exception:
         exc_type, exc, tb = sys.exc_info()
@@ -101,6 +131,14 @@ def _execute_task(task):
         send_task_notification(task)
 
 
+def running_from_thread_pool() -> bool:
+    # TODO: this needs an alternative approach ASAP!
+    # Currently we rely on the weak fact that ThreadPoolExecutor names threads like:
+    # "ThreadPoolExecutor-0_0"
+    thread_name = threading.current_thread().name
+    return "ThreadPoolExecutor" in thread_name
+
+
 def dispatch(
     func,
     args=None,
@@ -148,6 +186,8 @@ def dispatch(
         ValueError: When `resources` is an unsupported type.
     """
 
+    # Can't run short tasks immediately if running from thread pool
+    immediate = immediate and not running_from_thread_pool()
     assert deferred or immediate, "A task must be at least `deferred` or `immediate`."
 
     if callable(func):

pulpcore/tests/functional/api/test_tasking.py

@@ -10,8 +10,10 @@ from urllib.parse import urljoin
 from uuid import uuid4
 
 from pulpcore.client.pulpcore import ApiException
+from contextlib import contextmanager
 
-from pulpcore.tests.functional.utils import download_file
+from pulpcore.tests.functional.utils import download_file, PulpTaskError
+from pulpcore.constants import IMMEDIATE_TIMEOUT
 
 
 @pytest.fixture(scope="module")
@@ -445,3 +447,151 @@ def test_cancel_task_group(pulpcore_bindings, dispatch_task_group, gen_user):
 
     with gen_user(model_roles=["core.task_owner"]):
         pulpcore_bindings.TaskGroupsApi.task_groups_cancel(tgroup_href, {"state": "canceled"})
+
+
+LT_TIMEOUT = IMMEDIATE_TIMEOUT / 2
+GT_TIMEOUT = IMMEDIATE_TIMEOUT * 2
+
+
+class TestImmediateTaskWithNoResource:
+
+    @pytest.mark.parallel
+    def test_succeeds_on_api_worker(self, pulpcore_bindings, dispatch_task):
+        """
+        GIVEN a task with no resource requirements
+        AND the task IS an async function
+        WHEN dispatching a task as immediate
+        THEN the task completes with no associated worker
+        """
+        task_href = dispatch_task(
+            "pulpcore.app.tasks.test.asleep", args=(LT_TIMEOUT,), immediate=True
+        )
+        task = pulpcore_bindings.TasksApi.read(task_href)
+        assert task.state == "completed"
+        assert task.worker is None
+
+    @pytest.mark.parallel
+    def test_executes_on_api_worker_when_no_async(self, pulpcore_bindings, dispatch_task, capsys):
+        """
+        GIVEN a task with no resource requirements
+        AND the task IS NOT an async function
+        WHEN dispatching a task as immediate
+        THEN the task completes with no associated worker
+        """
+        # TODO: on 3.85 this should throw an error
+        task_href = dispatch_task(
+            "pulpcore.app.tasks.test.sleep", args=(LT_TIMEOUT,), immediate=True
+        )
+        stderr_content = capsys.readouterr().err
+        task = pulpcore_bindings.TasksApi.read(task_href)
+        assert task.state == "completed"
+        assert task.worker is None
+        assert "Support for non-coroutine immediate tasks will be dropped" in stderr_content
+
+    @pytest.mark.parallel
+    def test_timeouts_on_api_worker(self, pulpcore_bindings, dispatch_task):
+        """
+        GIVEN a task with no resource requirements
+        AND the task is an async function
+        WHEN dispatching a task as immediate
+        AND it takes longer than timeout
+        THEN the task fails with a timeout error message
+        """
+        task_href = dispatch_task(
+            "pulpcore.app.tasks.test.asleep", args=(GT_TIMEOUT,), immediate=True
+        )
+        task = pulpcore_bindings.TasksApi.read(task_href)
+        assert task.worker is None
+        assert "task timed out after" in task.error["description"]
+
+
+@pytest.fixture
+def resource_blocker(pulpcore_bindings, dispatch_task):
+
+    @contextmanager
+    def _resource_blocker(exclusive_resources: list[str], duration=20):
+        task_href = dispatch_task(
+            "pulpcore.app.tasks.test.sleep",
+            args=(duration,),
+            exclusive_resources=exclusive_resources,
+        )
+        yield
+        # Trying to cancel a finished task will return a 409 code.
+        # We can ignore if that's the case, because all we want here is to cut time down.
+        # Otherwise it might be a real error.
+        try:
+            pulpcore_bindings.TasksApi.tasks_cancel(task_href, {"state": "canceled"})
+        except ApiException as e:
+            if e.status != 409:
+                raise
+
+    return _resource_blocker
+
+
+class TestImmediateTaskWithBlockedResource:
+
+    @pytest.mark.parallel
+    def test_executes_in_task_worker(
+        self, resource_blocker, dispatch_task, monitor_task, pulpcore_bindings
+    ):
+        """
+        GIVEN an async task requiring busy resources
+        WHEN dispatching a task as immediate
+        THEN the task completes with a worker
+        """
+        COMMON_RESOURCE = str(uuid4())
+        with resource_blocker(exclusive_resources=[COMMON_RESOURCE]):
+            task_href = dispatch_task(
+                "pulpcore.app.tasks.test.asleep",
+                args=(LT_TIMEOUT,),
+                immediate=True,
+                exclusive_resources=[COMMON_RESOURCE],
+            )
+        task = monitor_task(task_href)
+        assert task.state == "completed"
+        assert task.worker is not None
+
+    @pytest.mark.parallel
+    def test_throws_when_non_deferrable(
+        self, resource_blocker, pulpcore_bindings, dispatch_task, monitor_task
+    ):
+        """
+        GIVEN an async task requiring busy resources
+        WHEN dispatching as immediate and not deferrable
+        THEN an error is raised
+        """
+        COMMON_RESOURCE = str(uuid4())
+        with resource_blocker(exclusive_resources=[COMMON_RESOURCE]):
+            task_href = dispatch_task(
+                "pulpcore.app.tasks.test.asleep",
+                args=(0,),
+                immediate=True,
+                deferred=False,
+                exclusive_resources=[COMMON_RESOURCE],
+            )
+            task = pulpcore_bindings.TasksApi.read(task_href)
+            assert task.state == "canceled"
+            assert task.worker is None
+            assert "Resources temporarily unavailable." in task.error["reason"]
+
+    @pytest.mark.parallel
+    def test_times_out_on_task_worker(
+        self, resource_blocker, pulpcore_bindings, dispatch_task, monitor_task
+    ):
+        """
+        GIVEN an async task requiring busy resources
+        WHEN dispatching a task as immediate
+        AND it takes longer than timeout
+        THEN an error is raised
+        """
+        COMMON_RESOURCE = str(uuid4())
+        with pytest.raises(PulpTaskError) as ctx:
+            with resource_blocker(exclusive_resources=[COMMON_RESOURCE]):
+                task_href = dispatch_task(
+                    "pulpcore.app.tasks.test.asleep",
+                    args=(GT_TIMEOUT,),
+                    immediate=True,
+                    exclusive_resources=[COMMON_RESOURCE],
+                )
+            monitor_task(task_href)
+        assert "task timed out after" in ctx.value.task.error["description"]

pulpcore/tests/functional/api/using_plugin/test_content_cache.py

@@ -16,7 +16,7 @@ from pulpcore.tests.functional.utils import get_from_url
 @pytest.mark.parallel
 def test_full_workflow(
     file_repo_with_auto_publish,
-    basic_manifest_path,
+    duplicate_filename_paths,
     file_remote_factory,
     file_bindings,
     distribution_base_url,
@@ -37,7 +37,8 @@ def test_full_workflow(
         return r.status, r.headers.get("X-PULP-CACHE")
 
     # Sync from the remote and assert that a new repository version is created
-    remote = file_remote_factory(manifest_path=basic_manifest_path, policy="immediate")
+    manifest_1, manifest_2 = duplicate_filename_paths
+    remote = file_remote_factory(manifest_path=manifest_1, policy="immediate")
     body = RepositorySyncURL(remote=remote.pulp_href)
     monitor_task(
         file_bindings.RepositoriesFileApi.sync(file_repo_with_auto_publish.pulp_href, body).task
@@ -129,6 +130,21 @@ def test_full_workflow(
         url = urljoin(distro_base_url, file)
         assert (200, "HIT" if i % 2 == 1 else "MISS") == _check_cache(url), file
 
+    # Sync a new remote with same filenames but on-demand
+    remote = file_remote_factory(manifest_path=manifest_2, policy="on_demand")
+    body = RepositorySyncURL(remote=remote.pulp_href)
+    monitor_task(
+        file_bindings.RepositoriesFileApi.sync(file_repo_with_auto_publish.pulp_href, body).task
+    )
+    repo = file_bindings.RepositoriesFileApi.read(file_repo_with_auto_publish.pulp_href)
+    assert repo.latest_version_href.endswith("/versions/3/")
+
+    # Test that cache is invalidated from sync, but on-demand responses are immediately cached
+    files = ["1.iso", "1.iso", "2.iso", "2.iso", "3.iso", "3.iso"]
+    for i, file in enumerate(files):
+        url = urljoin(distro_base_url, file)
+        assert (200, "HIT" if i % 2 == 1 else None) == _check_cache(url), file
+
     # Tests that deleting a repository invalidates the cache"""
     monitor_task(file_bindings.RepositoriesFileApi.delete(repo.pulp_href).task)
     files = ["", "PULP_MANIFEST", "2.iso"]

{pulpcore-3.76.1.dist-info → pulpcore-3.77.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulpcore
-Version: 3.76.1
+Version: 3.77.1
 Summary: Pulp Django Application and Related Modules
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -21,14 +21,14 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aiodns<=3.2.0,>=3.0
+Requires-Dist: aiodns<=3.3.0,>=3.0
 Requires-Dist: aiofiles<24.2.0,>=22.1
 Requires-Dist: aiohttp<3.12,>=3.8.1
 Requires-Dist: asyncio-throttle<=1.0.2,>=1.0
 Requires-Dist: async-timeout<4.0.4,>=4.0.3; python_version < "3.11"
 Requires-Dist: backoff<2.2.2,>=2.1.2
 Requires-Dist: click<=8.1.8,>=8.1.0
-Requires-Dist: cryptography<44.0.3,>=38.0.1
+Requires-Dist: cryptography<44.0.4,>=38.0.1
 Requires-Dist: Django~=4.2.0
 Requires-Dist: django-filter<=25.1,>=23.1
 Requires-Dist: django-guid<=3.5.1,>=3.3
@@ -52,13 +52,13 @@ Requires-Dist: opentelemetry-exporter-otlp-proto-http<1.33,>=1.27.0
 Requires-Dist: protobuf<6.0,>=4.21.1
 Requires-Dist: pulp-glue<0.33,>=0.18.0
 Requires-Dist: pygtrie<=2.5.0,>=2.5
-Requires-Dist: psycopg[binary]<=3.2.6,>=3.1.8
+Requires-Dist: psycopg[binary]<=3.2.7,>=3.1.8
 Requires-Dist: pyparsing<=3.2.3,>=3.1.0
 Requires-Dist: python-gnupg<=0.5.4,>=0.5
 Requires-Dist: PyYAML<=6.0.2,>=5.1.1
 Requires-Dist: redis<5.2.2,>=4.3
 Requires-Dist: tablib<3.6.0
-Requires-Dist: url-normalize<=2.2.0,>=1.4.3
+Requires-Dist: url-normalize<=2.2.1,>=1.4.3
 Requires-Dist: uuid6<=2024.7.10,>=2023.5.2
 Requires-Dist: whitenoise<6.10.0,>=5.0
 Requires-Dist: yarl<1.20.1,>=1.8
@@ -75,6 +75,9 @@ Requires-Dist: django-prometheus; extra == "prometheus"
 Provides-Extra: kafka
 Requires-Dist: cloudevents==1.11.0; extra == "kafka"
 Requires-Dist: confluent-kafka<2.10.0,>=2.4.0; extra == "kafka"
+Provides-Extra: diagnostics
+Requires-Dist: pyinstrument~=5.0; extra == "diagnostics"
+Requires-Dist: memray~=1.17; extra == "diagnostics"
 Dynamic: license-file