pulpcore 3.90.1__tar.gz → 3.91.0__tar.gz

{pulpcore-3.90.1 → pulpcore-3.91.0}/CHANGES.md

@@ -8,26 +8,31 @@
 
 [//]: # (towncrier release notes start)
 
-## 3.90.1 (2025-10-01) {: #3.90.1 }
+## 3.91.0 (2025-10-01) {: #3.91.0 }
 
-### REST API {: #3.90.1-rest-api }
+### REST API {: #3.91.0-rest-api }
 
-#### Bugfixes {: #3.90.1-rest-api-bugfix }
+#### Features {: #3.91.0-rest-api-feature }
+
+- Changed orphan cleanup to run as a scheduled task.
+- Removed the use of session scoped advisory locks from workers janitorial work.
+
+#### Bugfixes {: #3.91.0-rest-api-bugfix }
 
 - Fix situations with a single worker per installation failing to unblock all tasks that come in quick succession.
   [#6873](https://github.com/pulp/pulpcore/issues/6873)
 - Fixed exception propagation in the pulp-content instrumentation middleware.
   [#6925](https://github.com/pulp/pulpcore/issues/6925)
 
-### Plugin API {: #3.90.1-plugin-api }
+### Plugin API {: #3.91.0-plugin-api }
 
 No significant changes.
 
-### Pulp File {: #3.90.1-pulp-file }
+### Pulp File {: #3.91.0-pulp-file }
 
 No significant changes.
 
-### Pulp Cert Guard {: #3.90.1-pulp-cert-guard }
+### Pulp Cert Guard {: #3.91.0-pulp-cert-guard }
 
 No significant changes.
 

{pulpcore-3.90.1 → pulpcore-3.91.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulpcore
-Version: 3.90.1
+Version: 3.91.0
 Summary: Pulp Django Application and Related Modules
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulp_certguard/app/__init__.py

@@ -6,6 +6,6 @@ class PulpCertGuardPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_certguard.app"
     label = "certguard"
-    version = "3.90.1"
+    version = "3.91.0"
     python_package_name = "pulpcore"
     domain_compatible = True

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulp_file/app/__init__.py

@@ -8,6 +8,6 @@ class PulpFilePluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_file.app"
     label = "file"
-    version = "3.90.1"
+    version = "3.91.0"
     python_package_name = "pulpcore"
     domain_compatible = True

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore/app/apps.py

@@ -8,7 +8,7 @@ from django import apps
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db import connection, transaction
-from django.db.models.signals import post_migrate
+from django.db.models.signals import post_migrate, pre_migrate
 from django.utils.module_loading import module_has_submodule
 
 from pulpcore.exceptions.plugin import MissingPlugin
@@ -239,7 +239,7 @@ class PulpAppConfig(PulpPluginAppConfig):
     label = "core"
 
     # The version of this app
-    version = "3.90.1"
+    version = "3.91.0"
 
     # The python package name providing this app
     python_package_name = "pulpcore"
@@ -250,6 +250,7 @@ class PulpAppConfig(PulpPluginAppConfig):
         super().ready()
         from . import checks  # noqa
 
+        pre_migrate.connect(_clean_app_status, sender=self, dispatch_uid="clean_app_status")
         post_migrate.connect(
             _ensure_default_domain, sender=self, dispatch_uid="ensure_default_domain"
         )
@@ -263,6 +264,19 @@ class PulpAppConfig(PulpPluginAppConfig):
         )
 
 
+def _clean_app_status(sender, apps, verbosity, **kwargs):
+    from django.contrib.postgres.functions import TransactionNow
+    from django.db.models import F
+
+    try:
+        AppStatus = apps.get_model("core", "AppStatus")
+    except LookupError:
+        if verbosity >= 1:
+            print(_("AppStatus model does not exist. Skipping pre migrate cleanup."))
+    else:
+        AppStatus.objects.filter(last_heartbeat__lt=TransactionNow() - F("ttl")).delete()
+
+
 def _populate_access_policies(sender, apps, verbosity, **kwargs):
     from pulpcore.app.util import get_view_urlpattern
     from pulpcore.app.viewsets import LoginViewSet

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore/app/util.py

@@ -25,7 +25,6 @@ from rest_framework.reverse import reverse as drf_reverse
 from pulpcore.app.loggers import deprecation_logger
 from pulpcore.app.apps import pulp_plugin_configs
 from pulpcore.app import models
-from pulpcore.exceptions import AdvisoryLockError
 from pulpcore.exceptions.validation import InvalidSignatureError
 
 
@@ -468,6 +467,7 @@ def configure_cleanup():
             settings.TMPFILE_PROTECTION_TIME,
         ),
         ("tasks", "pulpcore.app.tasks.purge.purge", settings.TASK_PROTECTION_TIME),
+        ("content", "pulpcore.app.tasks.orphan.orphan_cleanup", settings.ORPHAN_PROTECTION_TIME),
     ]:
         if protection_time > 0:
             dispatch_interval = timedelta(minutes=protection_time)
@@ -636,34 +636,6 @@ def get_worker_name():
     return f"{os.getpid()}@{socket.gethostname()}"
 
 
-class PGAdvisoryLock:
-    """
-    A context manager that will hold a postgres advisory lock non-blocking.
-
-    The locks can be chosen from a lock group to avoid collisions. They will never collide with the
-    locks used for tasks.
-    """
-
-    def __init__(self, lock, lock_group=0):
-        self.lock_group = lock_group
-        self.lock = lock
-
-    def __enter__(self):
-        with connection.cursor() as cursor:
-            cursor.execute("SELECT pg_try_advisory_lock(%s, %s)", [self.lock_group, self.lock])
-            acquired = cursor.fetchone()[0]
-        if not acquired:
-            raise AdvisoryLockError("Could not acquire lock.")
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        with connection.cursor() as cursor:
-            cursor.execute("SELECT pg_advisory_unlock(%s, %s)", [self.lock_group, self.lock])
-            released = cursor.fetchone()[0]
-        if not released:
-            raise RuntimeError("Lock not held.")
-
-
 def normalize_http_status(status):
     """Convert the HTTP status code to 2xx, 3xx, etc., normalizing the last two digits."""
     if 100 <= status < 200:

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore/constants.py

@@ -6,11 +6,10 @@ from types import SimpleNamespace
 # The group will be 0.
 # The numbers are randomly chosen.
 # !!! Never change these values !!!
-TASK_DISPATCH_LOCK = 21
 TASK_SCHEDULING_LOCK = 42
 TASK_UNBLOCKING_LOCK = 84
-TASK_METRICS_HEARTBEAT_LOCK = 74
-STORAGE_METRICS_LOCK = 72
+TASK_METRICS_LOCK = 74
+WORKER_CLEANUP_LOCK = 11
 
 # Reasons to send along a task worker wakeup call.
 TASK_WAKEUP_UNBLOCK = "unblock"

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore/exceptions/__init__.py

@@ -1,5 +1,4 @@
 from .base import (
-    AdvisoryLockError,
     PulpException,
     ResourceImmutableError,
     TimeoutException,

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore/exceptions/base.py

@@ -64,10 +64,6 @@ class ResourceImmutableError(PulpException):
         return msg
 
 
-class AdvisoryLockError(Exception):
-    """Exception to signal that a lock could not be acquired."""
-
-
 class TimeoutException(PulpException):
     """
     Exception to signal timeout error.

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore/tasking/worker.py

@@ -1,19 +1,19 @@
 from gettext import gettext as _
 
+import functools
 import logging
 import os
 import random
 import select
 import signal
 import socket
-import contextlib
 from datetime import datetime, timedelta
 from multiprocessing import Process
 from tempfile import TemporaryDirectory
 from packaging.version import parse as parse_version
 
 from django.conf import settings
-from django.db import connection, DatabaseError, IntegrityError
+from django.db import connection, transaction, DatabaseError, IntegrityError
 from django.db.models import Case, Count, F, Max, Value, When
 from django.utils import timezone
 
@@ -22,15 +22,14 @@ from pulpcore.constants import (
     TASK_INCOMPLETE_STATES,
     TASK_SCHEDULING_LOCK,
     TASK_UNBLOCKING_LOCK,
-    TASK_METRICS_HEARTBEAT_LOCK,
+    TASK_METRICS_LOCK,
+    WORKER_CLEANUP_LOCK,
     TASK_WAKEUP_UNBLOCK,
     TASK_WAKEUP_HANDLE,
 )
 from pulpcore.metrics import init_otel_meter
 from pulpcore.app.apps import pulp_plugin_configs
 from pulpcore.app.models import Task, AppStatus
-from pulpcore.app.util import PGAdvisoryLock
-from pulpcore.exceptions import AdvisoryLockError
 
 from pulpcore.tasking.storage import WorkerDirectory
 from pulpcore.tasking._util import (
@@ -59,6 +58,30 @@ IGNORED_TASKS_CLEANUP_INTERVAL = 100
 THRESHOLD_UNBLOCKED_WAITING_TIME = 5
 
 
+def exclusive(lock):
+    """
+    Runs function in a transaction holding the specified lock.
+    Returns None if the lock could not be acquired.
+    It should be used for actions that only need to be performed by a single worker.
+    """
+
+    def _decorator(f):
+        @functools.wraps(f)
+        def _f(self, *args, **kwargs):
+            with transaction.atomic():
+                with connection.cursor() as cursor:
+                    cursor.execute("SELECT pg_try_advisory_xact_lock(%s, %s)", [0, lock])
+                    acquired = cursor.fetchone()[0]
+                if acquired:
+                    return f(self, *args, **kwargs)
+                else:
+                    return None
+
+        return _f
+
+    return _decorator
+
+
 class PulpcoreWorker:
     def __init__(self, auxiliary=False):
         # Notification states from several signal handlers
@@ -112,9 +135,9 @@ class PulpcoreWorker:
                 description="The age of the longest waiting task.",
                 unit="seconds",
             )
-            self.record_unblocked_waiting_tasks_metric = self._record_unblocked_waiting_tasks_metric
+            self.otel_enabled = True
         else:
-            self.record_unblocked_waiting_tasks_metric = lambda *args, **kwargs: None
+            self.otel_enabled = False
 
     def _signal_handler(self, thesignal, frame):
         if thesignal in (signal.SIGHUP, signal.SIGTERM):
@@ -187,7 +210,8 @@ class PulpcoreWorker:
         ):
             self.ignored_task_ids.remove(pk)
 
-    def worker_cleanup(self):
+    @exclusive(WORKER_CLEANUP_LOCK)
+    def app_worker_cleanup(self):
         qs = AppStatus.objects.missing()
         for app_worker in qs:
             _logger.warning(
@@ -198,8 +222,45 @@ class PulpcoreWorker:
         # Don't bother the others.
         self.wakeup_unblock = True
 
+    @exclusive(TASK_SCHEDULING_LOCK)
+    def dispatch_scheduled_tasks(self):
+        dispatch_scheduled_tasks()
+
+    @exclusive(TASK_METRICS_LOCK)
+    def record_unblocked_waiting_tasks_metric(self, now):
+        # This "reporting code" must not me moved inside a task, because it is supposed
+        # to be able to report on a congested tasking system to produce reliable results.
+        # For performance reasons we aggregate these statistics on a single database call.
+        unblocked_tasks_stats = (
+            Task.objects.filter(unblocked_at__isnull=False, state=TASK_STATES.WAITING)
+            .annotate(unblocked_for=Value(timezone.now()) - F("unblocked_at"))
+            .aggregate(
+                longest_unblocked_waiting_time=Max("unblocked_for", default=timezone.timedelta(0)),
+                unblocked_tasks_count_gte_threshold=Count(
+                    Case(
+                        When(
+                            unblocked_for__gte=Value(
+                                timezone.timedelta(seconds=THRESHOLD_UNBLOCKED_WAITING_TIME)
+                            ),
+                            then=1,
+                        )
+                    )
+                ),
+            )
+        )
+
+        self.tasks_unblocked_queue_meter.set(
+            unblocked_tasks_stats["unblocked_tasks_count_gte_threshold"]
+        )
+        self.tasks_longest_unblocked_time_meter.set(
+            unblocked_tasks_stats["longest_unblocked_waiting_time"].seconds
+        )
+
+        self.cursor.execute(f"NOTIFY pulp_worker_metrics_heartbeat, '{str(now)}'")
+
     def beat(self):
-        if self.app_status.last_heartbeat < timezone.now() - self.heartbeat_period:
+        now = timezone.now()
+        if self.app_status.last_heartbeat < now - self.heartbeat_period:
             self.handle_worker_heartbeat()
             if self.ignored_task_ids:
                 self.ignored_task_countdown -= 1
@@ -210,12 +271,12 @@ class PulpcoreWorker:
                 self.worker_cleanup_countdown -= 1
                 if self.worker_cleanup_countdown <= 0:
                     self.worker_cleanup_countdown = WORKER_CLEANUP_INTERVAL
-                    self.worker_cleanup()
-                with contextlib.suppress(AdvisoryLockError), PGAdvisoryLock(TASK_SCHEDULING_LOCK):
-                    dispatch_scheduled_tasks()
-                # This "reporting code" must not me moved inside a task, because it is supposed
-                # to be able to report on a congested tasking system to produce reliable results.
-                self.record_unblocked_waiting_tasks_metric()
+                    self.app_worker_cleanup()
+
+                self.dispatch_scheduled_tasks()
+
+                if self.otel_enabled and now > self.last_metric_heartbeat + self.heartbeat_period:
+                    self.record_unblocked_waiting_tasks_metric(now)
 
     def notify_workers(self, reason):
         self.cursor.execute("SELECT pg_notify('pulp_worker_wakeup', %s)", (reason,))
@@ -283,19 +344,20 @@ class PulpcoreWorker:
         assert not self.auxiliary
 
         self.wakeup_unblock = False
-        with contextlib.suppress(AdvisoryLockError), PGAdvisoryLock(TASK_UNBLOCKING_LOCK):
-            self._unblock_tasks()
+        result = self._unblock_tasks()
+        if result is not None and (
+            Task.objects.filter(
+                state__in=[TASK_STATES.WAITING, TASK_STATES.CANCELING], app_lock=None
+            )
+            .exclude(unblocked_at=None)
+            .exists()
+        ):
+            self.notify_workers(TASK_WAKEUP_HANDLE)
+            return True
 
-            if (
-                Task.objects.filter(state__in=TASK_INCOMPLETE_STATES, app_lock=None)
-                .exclude(unblocked_at=None)
-                .exists()
-            ):
-                self.notify_workers(TASK_WAKEUP_HANDLE)
-                return True
-            return False
-        return None
+        return result
 
+    @exclusive(TASK_UNBLOCKING_LOCK)
     def _unblock_tasks(self):
         """Iterate over waiting tasks and mark them unblocked accordingly."""
 
@@ -359,6 +421,7 @@ class PulpcoreWorker:
             # Record the resources of the pending task
             taken_exclusive_resources.update(exclusive_resources)
             taken_shared_resources.update(shared_resources)
+        return False
 
     def sleep(self):
         """Wait for signals on the wakeup channel while heart beating."""
@@ -545,42 +608,6 @@ class PulpcoreWorker:
                 if rows != 1:
                     raise RuntimeError("Something other than us is messing around with locks.")
 
-    def _record_unblocked_waiting_tasks_metric(self):
-        now = timezone.now()
-        if now > self.last_metric_heartbeat + self.heartbeat_period:
-            with contextlib.suppress(AdvisoryLockError), PGAdvisoryLock(
-                TASK_METRICS_HEARTBEAT_LOCK
-            ):
-                # For performance reasons we aggregate these statistics on a single database call.
-                unblocked_tasks_stats = (
-                    Task.objects.filter(unblocked_at__isnull=False, state=TASK_STATES.WAITING)
-                    .annotate(unblocked_for=Value(timezone.now()) - F("unblocked_at"))
-                    .aggregate(
-                        longest_unblocked_waiting_time=Max(
-                            "unblocked_for", default=timezone.timedelta(0)
-                        ),
-                        unblocked_tasks_count_gte_threshold=Count(
-                            Case(
-                                When(
-                                    unblocked_for__gte=Value(
-                                        timezone.timedelta(seconds=THRESHOLD_UNBLOCKED_WAITING_TIME)
-                                    ),
-                                    then=1,
-                                )
-                            )
-                        ),
-                    )
-                )
-
-                self.tasks_unblocked_queue_meter.set(
-                    unblocked_tasks_stats["unblocked_tasks_count_gte_threshold"]
-                )
-                self.tasks_longest_unblocked_time_meter.set(
-                    unblocked_tasks_stats["longest_unblocked_waiting_time"].seconds
-                )
-
-                self.cursor.execute(f"NOTIFY pulp_worker_metrics_heartbeat, '{str(now)}'")
-
     def run(self, burst=False):
         with WorkerDirectory(self.name):
             signal.signal(signal.SIGINT, self._signal_handler)

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py

@@ -8,7 +8,6 @@ the case.
 import pytest
 import uuid
 
-from pulpcore.app import settings
 from pulpcore.client.pulpcore.exceptions import ApiException
 
 from pulpcore.client.pulp_file import RepositorySyncURL
@@ -16,8 +15,6 @@ from pulpcore.client.pulp_file import RepositorySyncURL
 NUM_REPOS = 1
 NUM_EXPORTERS = 4
 
-pytestmark = pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support export.")
-
 
 @pytest.fixture
 def create_exporter(pulpcore_bindings, gen_object_with_cleanup, add_to_filesystem_cleanup):

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py

@@ -21,7 +21,6 @@ NUM_REPOS = 2
 
 
 pytestmark = [
-    pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support import."),
     pytest.mark.skipif(
         "/tmp" not in settings.ALLOWED_IMPORT_PATHS,
         reason="Cannot run import-tests unless /tmp is in ALLOWED_IMPORT_PATHS ({}).".format(

{pulpcore-3.90.1 → pulpcore-3.91.0}/pulpcore.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulpcore
-Version: 3.90.1
+Version: 3.91.0
 Summary: Pulp Django Application and Related Modules
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org

{pulpcore-3.90.1 → pulpcore-3.91.0}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = "pulpcore"
-version = "3.90.1"
+version = "3.91.0"
 description = "Pulp Django Application and Related Modules"
 readme = "README.md"
 authors = [
@@ -204,7 +204,7 @@ ignore = [
 [tool.bumpversion]
 # This section is managed by the plugin template. Do not edit manually.
 
-current_version = "3.90.1"
+current_version = "3.91.0"
 commit = false
 tag = false
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<alpha>0a)?(?P<patch>\\d+)(\\.(?P<release>[a-z]+))?"