pulpcore 3.83.1__tar.gz → 3.84.0__tar.gz

{pulpcore-3.83.1 → pulpcore-3.84.0}/CHANGES.md

@@ -8,6 +8,61 @@
 
 [//]: # (towncrier release notes start)
 
+## 3.84.0 (2025-07-30) {: #3.84.0 }
+
+### REST API {: #3.84.0-rest-api }
+
+#### Features {: #3.84.0-rest-api-feature }
+
+- Added the `--auxiliary` flag to the workers entrypoint to allow adding workers to an installation that will only handle unblocked tasks.
+- Moved timestamping of new tasks from application code into the database.
+
+#### Bugfixes {: #3.84.0-rest-api-bugfix }
+
+- Fixed the new content set optimization failing when the RepositoryVersion grew larger than 65K content units.
+  [#6772](https://github.com/pulp/pulpcore/issues/6772)
+- Add some randomness to the task pickup code to relieve some pressure from the queue.
+  [#6799](https://github.com/pulp/pulpcore/issues/6799)
+- When running as a recurring scheduled task, purge will now clean stale task records accross all domains.
+  [#6815](https://github.com/pulp/pulpcore/issues/6815)
+- Added some Postgresql transactions in places where they are appropriate.
+
+### Plugin API {: #3.84.0-plugin-api }
+
+No significant changes.
+
+### Pulp File {: #3.84.0-pulp-file }
+
+No significant changes.
+
+### Pulp Cert Guard {: #3.84.0-pulp-cert-guard }
+
+No significant changes.
+
+---
+
+## 3.83.2 (2025-07-17) {: #3.83.2 }
+
+### REST API {: #3.83.2-rest-api }
+
+#### Bugfixes {: #3.83.2-rest-api-bugfix }
+
+- Added some Postgresql transactions in places where they are appropriate.
+
+### Plugin API {: #3.83.2-plugin-api }
+
+No significant changes.
+
+### Pulp File {: #3.83.2-pulp-file }
+
+No significant changes.
+
+### Pulp Cert Guard {: #3.83.2-pulp-cert-guard }
+
+No significant changes.
+
+---
+
 ## 3.83.1 (2025-07-15) {: #3.83.1 }
 
 ### REST API {: #3.83.1-rest-api }

{pulpcore-3.83.1 → pulpcore-3.84.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulpcore
-Version: 3.83.1
+Version: 3.84.0
 Summary: Pulp Django Application and Related Modules
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -44,11 +44,11 @@ Requires-Dist: gunicorn<23.1.0,>=22.0
 Requires-Dist: importlib-metadata<=6.0.1,>=6.0.1
 Requires-Dist: jinja2<=3.1.6,>=3.1
 Requires-Dist: json_stream<2.4,>=2.3.2
-Requires-Dist: jq<1.10.0,>=1.6.0
+Requires-Dist: jq<1.11.0,>=1.6.0
 Requires-Dist: PyOpenSSL<26.0
-Requires-Dist: opentelemetry-api<1.35,>=1.27.0
-Requires-Dist: opentelemetry-sdk<1.35,>=1.27.0
-Requires-Dist: opentelemetry-exporter-otlp-proto-http<1.35,>=1.27.0
+Requires-Dist: opentelemetry-api<1.36,>=1.27.0
+Requires-Dist: opentelemetry-sdk<1.36,>=1.27.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<1.36,>=1.27.0
 Requires-Dist: protobuf<6.0,>=4.21.1
 Requires-Dist: pulp-glue<0.35,>=0.28.0
 Requires-Dist: pygtrie<=2.5.0,>=2.5
@@ -70,6 +70,7 @@ Provides-Extra: google
 Requires-Dist: django-storages[google]==1.14.6; extra == "google"
 Provides-Extra: azure
 Requires-Dist: django-storages[azure]==1.14.6; extra == "azure"
+Requires-Dist: azure-storage-blob<12.26.0; extra == "azure"
 Provides-Extra: prometheus
 Requires-Dist: django-prometheus; extra == "prometheus"
 Provides-Extra: kafka

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulp_certguard/app/__init__.py

@@ -6,6 +6,6 @@ class PulpCertGuardPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_certguard.app"
     label = "certguard"
-    version = "3.83.1"
+    version = "3.84.0"
     python_package_name = "pulpcore"
     domain_compatible = True

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulp_file/app/__init__.py

@@ -8,6 +8,6 @@ class PulpFilePluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_file.app"
     label = "file"
-    version = "3.83.1"
+    version = "3.84.0"
     python_package_name = "pulpcore"
     domain_compatible = True

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulpcore/app/apps.py

@@ -247,7 +247,7 @@ class PulpAppConfig(PulpPluginAppConfig):
     label = "core"
 
     # The version of this app
-    version = "3.83.1"
+    version = "3.84.0"
 
     # The python package name providing this app
     python_package_name = "pulpcore"

pulpcore-3.84.0/pulpcore/app/migrations/0134_task_insert_trigger.py

@@ -0,0 +1,81 @@
+# Generated by Django 4.2.22 on 2025-06-06 14:32
+
+from django.db import migrations
+from pulpcore.migrations import RequireVersion
+
+# This migration is supposed to be zero downtime upgrade safe.
+
+
+TASK_INSERT_TRIGGER_UP = """
+CREATE FUNCTION on_insert_timestamp_task()
+  RETURNS TRIGGER
+  LANGUAGE plpgsql
+  AS $$
+    DECLARE
+      res_list text[];
+      res_list_shared text[];
+    BEGIN
+      res_list := array_agg(q.res) FROM (SELECT regexp_replace(unnest(NEW.reserved_resources_record), '^shared:', '') res) AS q;
+      res_list_shared := array_agg(q.res) FROM (SELECT 'shared:' || unnest(res_list) AS res) AS q;
+      PERFORM pg_advisory_xact_lock(4711, q.id) FROM (SELECT hashtext(res) AS id FROM unnest(res_list) AS res ORDER BY id) AS q;
+      NEW.pulp_created = clock_timestamp();
+      IF NEW.pulp_created <= (
+        SELECT max(pulp_created) FROM core_task
+        WHERE
+          state NOT IN ('completed', 'failed', 'canceled', 'skipped')
+        AND
+          reserved_resources_record && (res_list || res_list_shared)
+      )
+      THEN
+        RAISE EXCEPTION 'Clock screw detected.';
+      END IF;
+      RETURN NEW;
+    END;
+  $$
+;
+
+CREATE FUNCTION on_update_timestamp_task()
+  RETURNS TRIGGER
+  LANGUAGE plpgsql
+  AS $$
+    BEGIN
+      -- This is mainly a safeguard to prevent old code from messing with the timestamp, now that the database is in charge.
+      RAISE EXCEPTION 'Updating pulp_created is not allowed.';
+    END;
+  $$
+;
+
+CREATE TRIGGER on_insert_timestamp_task_trigger
+  BEFORE INSERT
+  ON core_task
+  FOR EACH ROW
+  EXECUTE FUNCTION on_insert_timestamp_task()
+;
+
+CREATE TRIGGER on_update_timestamp_task_trigger
+  BEFORE UPDATE
+  ON core_task
+  FOR EACH ROW
+  WHEN (OLD.pulp_created <> NEW.pulp_created)
+  EXECUTE FUNCTION on_update_timestamp_task()
+;
+"""
+
+TASK_INSERT_TRIGGER_DOWN = """
+DROP TRIGGER on_update_timestamp_task_trigger on core_task;
+DROP TRIGGER on_insert_timestamp_task_trigger on core_task;
+DROP FUNCTION on_update_timestamp_task;
+DROP FUNCTION on_insert_timestamp_task;
+"""
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0133_repositoryversion_content_ids"),
+    ]
+
+    operations = [
+        RequireVersion("core", "3.82.1"),
+        migrations.RunSQL(sql=TASK_INSERT_TRIGGER_UP, reverse_sql=TASK_INSERT_TRIGGER_DOWN),
+    ]

pulpcore-3.84.0/pulpcore/app/migrations/0135_task_pulp_task_resources_index.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.22 on 2025-07-07 11:07
+
+import django.contrib.postgres.indexes
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0134_task_insert_trigger"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="task",
+            index=django.contrib.postgres.indexes.GinIndex(
+                condition=models.Q(
+                    ("state__in", ["completed", "failed", "canceled", "skipped"]), _negated=True
+                ),
+                fields=["reserved_resources_record"],
+                name="pulp_task_resources_index",
+                opclasses=["array_ops"],
+            ),
+        ),
+    ]

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulpcore/app/models/repository.py

@@ -1064,28 +1064,29 @@ class RepositoryVersion(BaseModel):
         )
         repo_content = []
         to_add = set(content.values_list("pk", flat=True)) - set(self._get_content_ids())
-        if to_add:
-            self.content_ids += list(to_add)
-            self.save()
-
-        # Normalize representation if content has already been removed in this version and
-        # is re-added: Undo removal by setting version_removed to None.
-        for removed in batch_qs(self.removed().order_by("pk").values_list("pk", flat=True)):
-            to_readd = to_add.intersection(set(removed))
-            if to_readd:
-                RepositoryContent.objects.filter(
-                    content__in=to_readd, repository=self.repository, version_removed=self
-                ).update(version_removed=None)
-                to_add = to_add - to_readd
-
-        for content_pk in to_add:
-            repo_content.append(
-                RepositoryContent(
-                    repository=self.repository, content_id=content_pk, version_added=self
+        with transaction.atomic():
+            if to_add:
+                self.content_ids += list(to_add)
+                self.save()
+
+            # Normalize representation if content has already been removed in this version and
+            # is re-added: Undo removal by setting version_removed to None.
+            for removed in batch_qs(self.removed().order_by("pk").values_list("pk", flat=True)):
+                to_readd = to_add.intersection(set(removed))
+                if to_readd:
+                    RepositoryContent.objects.filter(
+                        content__in=to_readd, repository=self.repository, version_removed=self
+                    ).update(version_removed=None)
+                    to_add = to_add - to_readd
+
+            for content_pk in to_add:
+                repo_content.append(
+                    RepositoryContent(
+                        repository=self.repository, content_id=content_pk, version_added=self
+                    )
                 )
-            )
 
-        RepositoryContent.objects.bulk_create(repo_content)
+            RepositoryContent.objects.bulk_create(repo_content)
 
     def remove_content(self, content):
         """
@@ -1111,23 +1112,24 @@ class RepositoryVersion(BaseModel):
         )
         content_ids = set(self._get_content_ids())
         to_remove = set(content.values_list("pk", flat=True))
-        if to_remove:
-            self.content_ids = list(content_ids - to_remove)
-            self.save()
-
-        # Normalize representation if content has already been added in this version.
-        # Undo addition by deleting the RepositoryContent.
-        RepositoryContent.objects.filter(
-            repository=self.repository,
-            content_id__in=content,
-            version_added=self,
-            version_removed=None,
-        ).delete()
-
-        q_set = RepositoryContent.objects.filter(
-            repository=self.repository, content_id__in=content, version_removed=None
-        )
-        q_set.update(version_removed=self)
+        with transaction.atomic():
+            if to_remove:
+                self.content_ids = list(content_ids - to_remove)
+                self.save()
+
+            # Normalize representation if content has already been added in this version.
+            # Undo addition by deleting the RepositoryContent.
+            RepositoryContent.objects.filter(
+                repository=self.repository,
+                content_id__in=content,
+                version_added=self,
+                version_removed=None,
+            ).delete()
+
+            q_set = RepositoryContent.objects.filter(
+                repository=self.repository, content_id__in=content, version_removed=None
+            )
+            q_set.update(version_removed=self)
 
     def set_content(self, content):
         """

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulpcore/app/models/task.py

@@ -9,6 +9,7 @@ from gettext import gettext as _
 
 from django.conf import settings
 from django.contrib.postgres.fields import ArrayField, HStoreField
+from django.contrib.postgres.indexes import GinIndex
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import connection, models
 from django.utils import timezone
@@ -348,6 +349,12 @@ class Task(BaseModel, AutoAddObjPermsMixin):
             models.Index(fields=["unblocked_at"]),
             models.Index(fields=["state"]),
             models.Index(fields=["state", "pulp_created"]),
+            GinIndex(
+                name="pulp_task_resources_index",
+                fields=["reserved_resources_record"],
+                condition=~models.Q(state__in=["completed", "failed", "canceled", "skipped"]),
+                opclasses=["array_ops"],
+            ),
         ]
         permissions = [
             ("manage_roles_task", "Can manage role assignments on task"),

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulpcore/app/settings.py

@@ -20,7 +20,8 @@ from django.core.files.storage import storages
 from django.conf import global_settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db import connection
-from dynaconf import DjangoDynaconf, Dynaconf, Validator
+from pulpcore.app.loggers import deprecation_logger
+from dynaconf import DjangoDynaconf, Dynaconf, Validator, get_history
 
 from pulpcore import constants
 
@@ -553,7 +554,14 @@ settings = DjangoDynaconf(
 
 # begin compatibility layer for DEFAULT_FILE_STORAGE
 # Remove on pulpcore=3.85 or pulpcore=4.0
-
+using_deprecated_storage_settings = len(get_history(settings, key="DEFAULT_FILE_STORAGE")) > 1
+if using_deprecated_storage_settings:
+    deprecation_logger.warning(
+        "[deprecation] DEFAULT_FILE_STORAGE will be removed in pulpcore 3.85. "
+        "Learn how to upgrade to STORAGES:\n"
+        "https://discourse.pulpproject.org/t/"
+        "action-required-upgrade-your-storage-settings-before-pulpcore-3-85/2072/2"
+    )
 # Ensures the cached property storage.backends uses the the right value
 storages._backends = settings.STORAGES.copy()
 storages.backends

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulpcore/app/tasks/purge.py

@@ -82,17 +82,20 @@ def purge(finished_before=None, states=None, **kwargs):
         states (Optional[List[str]]): List of task-states we want to purge.
 
     """
+    scheduled = current_task.get().taskschedule_set.exists()
+
     if finished_before is None:
         assert settings.TASK_PROTECTION_TIME > 0
         finished_before = timezone.now() - timezone.timedelta(minutes=settings.TASK_PROTECTION_TIME)
     if states is None:
         states = TASK_FINAL_STATES
-    domain = get_domain()
+
     # Tasks, prior to the specified date, in the specified state, owned by the current-user, in the
     # current domain
-    candidate_qs = Task.objects.filter(
-        finished_at__lt=finished_before, state__in=states, pulp_domain=domain
-    )
+    candidate_qs = Task.objects.filter(finished_at__lt=finished_before, state__in=states)
+
+    if not scheduled:
+        candidate_qs = candidate_qs.filter(pulp_domain=get_domain())
     if "user_pk" in kwargs:
         if (user_pk := kwargs["user_pk"]) is not None:
             current_user = User.objects.get(pk=user_pk)
@@ -101,7 +104,7 @@ def purge(finished_before=None, states=None, **kwargs):
         # This is the old task signature (<= 3.74) without "user_pk".
         # Has this task not been dispatched from a task schedule? Then we assume there was a user
         # doing that.
-        if not current_task.get().taskschedule_set.exists():
+        if not scheduled:
             current_user = get_current_authenticated_user()
             if current_user is None:
                 raise RuntimeError(

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulpcore/constants.py

@@ -12,6 +12,10 @@ TASK_UNBLOCKING_LOCK = 84
 TASK_METRICS_HEARTBEAT_LOCK = 74
 STORAGE_METRICS_LOCK = 72
 
+# Reasons to send along a task worker wakeup call.
+TASK_WAKEUP_UNBLOCK = "unblock"
+TASK_WAKEUP_HANDLE = "handle"
+
 #: All valid task states.
 TASK_STATES = SimpleNamespace(
     WAITING="waiting",

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulpcore/tasking/entrypoint.py

@@ -21,8 +21,18 @@ _logger = logging.getLogger(__name__)
 @click.option(
     "--reload/--no-reload", help="Reload worker on code changes. [requires hupper to be installed.]"
 )
+@click.option(
+    "--auxiliary/--no-auxiliary",
+    default=False,
+    help="Auxiliary workers do not perform housekeeping duties.",
+)
 @click.command()
-def worker(pid, burst, reload):
+def worker(
+    pid,
+    burst,
+    reload,
+    auxiliary,
+):
     """A Pulp worker."""
 
     if reload:
@@ -40,4 +50,4 @@ def worker(pid, burst, reload):
 
     _logger.info("Starting distributed type worker")
 
-    PulpcoreWorker().run(burst=burst)
+    PulpcoreWorker(auxiliary=auxiliary).run(burst=burst)

{pulpcore-3.83.1 → pulpcore-3.84.0}/pulpcore/tasking/tasks.py

@@ -9,12 +9,11 @@ import traceback
 import tempfile
 import threading
 from asgiref.sync import sync_to_async
-from datetime import timedelta
 from gettext import gettext as _
 
 from django.conf import settings
 from django.db import connection, transaction
-from django.db.models import Model, Max
+from django.db.models import Model
 from django_guid import get_guid
 from pulpcore.app.apps import MODULE_PLUGIN_VERSIONS
 from pulpcore.app.models import Task, TaskGroup
@@ -23,8 +22,8 @@ from pulpcore.constants import (
     TASK_FINAL_STATES,
     TASK_INCOMPLETE_STATES,
     TASK_STATES,
-    TASK_DISPATCH_LOCK,
     IMMEDIATE_TIMEOUT,
+    TASK_WAKEUP_UNBLOCK,
 )
 from pulpcore.middleware import x_task_diagnostics_var
 from pulpcore.tasking.kafka import send_task_notification
@@ -47,10 +46,10 @@ def _validate_and_get_resources(resources):
     return list(resource_set)
 
 
-def wakeup_worker():
+def wakeup_worker(reason="unknown"):
     # Notify workers
     with connection.connection.cursor() as cursor:
-        cursor.execute("NOTIFY pulp_worker_wakeup")
+        cursor.execute("SELECT pg_notify('pulp_worker_wakeup', %s)", (reason,))
 
 
 def execute_task(task):
@@ -228,13 +227,6 @@ def dispatch(
     notify_workers = False
     with contextlib.ExitStack() as stack:
         with transaction.atomic():
-            # Task creation need to be serialized so that pulp_created will provide a stable order
-            # at every time. We specifically need to ensure that each task, when committed to the
-            # task table will be the newest with respect to `pulp_created`.
-            with connection.cursor() as cursor:
-                # Wait for exclusive access and release automatically after transaction.
-                cursor.execute("SELECT pg_advisory_xact_lock(%s, %s)", [0, TASK_DISPATCH_LOCK])
-            newest_created = Task.objects.aggregate(Max("pulp_created"))["pulp_created__max"]
             task = Task.objects.create(
                 state=TASK_STATES.WAITING,
                 logging_cid=(get_guid()),
@@ -250,23 +242,6 @@ def dispatch(
                 profile_options=x_task_diagnostics_var.get(None),
             )
             task.refresh_from_db()  # The database may have assigned a timestamp for us.
-            if newest_created and task.pulp_created <= newest_created:
-                # Let this workaround not row forever into the future.
-                if newest_created - task.pulp_created > timedelta(seconds=1):
-                    # Do not commit the transaction if this condition is not met.
-                    # If we ever hit this, think about delegating the timestamping to PostgresQL.
-                    raise RuntimeError("Clockscrew detected. Task dispatching would be dangerous.")
-                # Try to work around the smaller glitch
-                task.pulp_created = newest_created + timedelta(milliseconds=1)
-                task.save()
-            if task_group:
-                task_group.refresh_from_db()
-                if task_group.all_tasks_dispatched:
-                    task.set_canceling()
-                    task.set_canceled(
-                        TASK_STATES.CANCELED, "All tasks in group have been dispatched/canceled."
-                    )
-                    return task
             if immediate:
                 # Grab the advisory lock before the task hits the db.
                 stack.enter_context(task)
@@ -308,7 +283,7 @@ def dispatch(
                 task.set_canceling()
                 task.set_canceled(TASK_STATES.CANCELED, "Resources temporarily unavailable.")
     if notify_workers:
-        wakeup_worker()
+        wakeup_worker(TASK_WAKEUP_UNBLOCK)
     return task