pulpcore 3.115.2__tar.gz → 3.115.4__tar.gz

{pulpcore-3.115.2 → pulpcore-3.115.4}/CHANGES.md

@@ -8,6 +8,56 @@
 
 [//]: # (towncrier release notes start)
 
+## 3.115.4 (2026-08-12) {: #3.115.4 }
+
+### REST API {: #3.115.4-rest-api }
+
+#### Bugfixes {: #3.115.4-rest-api-bugfix }
+
+- Reduced lock contention for distribution updates that leave `base_path` unchanged.
+  [#3322](https://github.com/pulp/pulpcore/issues/3322),
+  [#7896](https://github.com/pulp/pulpcore/issues/7896)
+- Fixed the `waiting_tasks` metric to count only tasks that can run in parallel under resource locks, matching worker FIFO scheduling.
+  [#7938](https://github.com/pulp/pulpcore/issues/7938)
+- Fixed Redis workers polling the database at the single-worker rate during startup by learning the online worker count before the first heartbeat.
+
+### Plugin API {: #3.115.4-plugin-api }
+
+No significant changes.
+
+### Pulp File {: #3.115.4-pulp-file }
+
+No significant changes.
+
+### Pulp Cert Guard {: #3.115.4-pulp-cert-guard }
+
+No significant changes.
+
+---
+
+## 3.115.3 (2026-07-28) {: #3.115.3 }
+
+### REST API {: #3.115.3-rest-api }
+
+#### Bugfixes {: #3.115.3-rest-api-bugfix }
+
+- Fixed domain list returning all domains to authenticated users instead of only domains they can view.
+  [#7898](https://github.com/pulp/pulpcore/issues/7898)
+
+### Plugin API {: #3.115.3-plugin-api }
+
+No significant changes.
+
+### Pulp File {: #3.115.3-pulp-file }
+
+No significant changes.
+
+### Pulp Cert Guard {: #3.115.3-pulp-cert-guard }
+
+No significant changes.
+
+---
+
 ## 3.115.2 (2026-07-22) {: #3.115.2 }
 
 ### REST API {: #3.115.2-rest-api }

{pulpcore-3.115.2 → pulpcore-3.115.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulpcore
-Version: 3.115.2
+Version: 3.115.4
 Summary: Pulp Django Application and Related Modules
 Author-email: Pulp Team <pulp-list@redhat.com>
 License-Expression: GPL-2.0-or-later

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulp_certguard/app/__init__.py

@@ -6,6 +6,6 @@ class PulpCertGuardPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_certguard.app"
     label = "certguard"
-    version = "3.115.2"
+    version = "3.115.4"
     python_package_name = "pulpcore"
     domain_compatible = True

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulp_file/app/__init__.py

@@ -8,6 +8,6 @@ class PulpFilePluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_file.app"
     label = "file"
-    version = "3.115.2"
+    version = "3.115.4"
     python_package_name = "pulpcore"
     domain_compatible = True

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore/app/apps.py

@@ -237,7 +237,7 @@ class PulpAppConfig(PulpPluginAppConfig):
     label = "core"
 
     # The version of this app
-    version = "3.115.2"
+    version = "3.115.4"
 
     # The python package name providing this app
     python_package_name = "pulpcore"

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore/app/viewsets/base.py

@@ -451,6 +451,14 @@ class AsyncReservedObjectMixin:
         ).format(self.__class__.__name__)
         return [instance]
 
+    def async_shared_resources(self, instance, **kwargs):
+        """
+        Return shared resources to reserve for the task created by the Async...Mixins.
+
+        This default implementation does not add any shared reservations.
+        """
+        return []
+
 
 class AsyncCreateMixin:
     """
@@ -472,6 +480,7 @@ class AsyncCreateMixin:
         task = dispatch(
             tasks.base.general_create,
             exclusive_resources=self.async_reserved_resources(None),
+            shared_resources=self.async_shared_resources(None),
             args=(app_label, serializer.__class__.__name__),
             kwargs=task_kwargs,
         )
@@ -506,6 +515,7 @@ class AsyncUpdateMixin(AsyncReservedObjectMixin):
             task = dispatch(
                 tasks.base.ageneral_update,
                 exclusive_resources=self.async_reserved_resources(instance),
+                shared_resources=self.async_shared_resources(instance),
                 args=(pk, app_label, serializer.__class__.__name__),
                 kwargs=task_kwargs,
                 immediate=self.ALLOW_NON_BLOCKING_UPDATE,
@@ -542,6 +552,7 @@ class AsyncRemoveMixin(AsyncReservedObjectMixin):
         task = dispatch(
             tasks.base.ageneral_delete,
             exclusive_resources=self.async_reserved_resources(instance),
+            shared_resources=self.async_shared_resources(instance),
             args=(pk, app_label, serializer.__class__.__name__),
             immediate=self.ALLOW_NON_BLOCKING_DELETE,
         )

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore/app/viewsets/domain.py

@@ -49,6 +49,7 @@ class DomainViewSet(
     serializer_class = DomainSerializer
     endpoint_name = "domains"
     filterset_class = DomainFilter
+    queryset_filtering_required_permission = "core.view_domain"
     DEFAULT_ACCESS_POLICY = {
         "statements": [
             {

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore/app/viewsets/publication.py

@@ -527,7 +527,31 @@ class BaseDistributionViewSet(NamedModelViewSet):
         return qs
 
     def async_reserved_resources(self, instance):
-        """Return resource that locks all Distributions."""
+        """
+        Reserve safe distribution locks for async operations.
+
+        The explicit distribution.base_path lock protects the domain-wide base_path invariant.
+        The older domain-scoped distributions lock remains shared so tasks queued before an upgrade
+        still overlap safely with new tasks.
+        """
+        distribution_base_path = f"pdrn:{get_domain().pulp_id}:distribution.base_path"
+        if instance is None:
+            return [distribution_base_path]
+
+        if getattr(self, "action", "") == "destroy":
+            return [instance]
+
+        request_data = getattr(getattr(self, "request", None), "data", {})
+        requested_base_path = request_data.get("base_path", instance.base_path)
+        if requested_base_path == instance.base_path:
+            return [instance]
+
+        return [instance, distribution_base_path]
+
+    def async_shared_resources(self, instance):
+        """
+        Keep the legacy domain-scoped distribution lock shared for upgrade compatibility.
+        """
         return [f"pdrn:{get_domain().pulp_id}:distributions"]
 
 
@@ -567,9 +591,9 @@ class DistributionViewSet(
     LabelsMixin,
 ):
     """
-    Provides read and list methods and also provides asynchronous CUD methods to dispatch tasks
-    with reservation that lock all Distributions preventing race conditions during base_path
-    checking.
+    Provides read and list methods plus asynchronous CUD methods that reserve per-distribution
+    locks, an explicit base_path lock when needed, and the legacy domain-wide distributions lock in
+    shared mode for upgrade compatibility.
     """
 
 

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore/tasking/redis_worker.py

@@ -93,6 +93,57 @@ def exclusive(lock):
     return _decorator
 
 
+def count_waiting_tasks_for_metric():
+    """
+    Count WAITING/RUNNING tasks older than five seconds that can run at the same
+    time given exclusive/shared resource reservations (FIFO-aware).
+
+    Returns:
+        int: Parallelizable unfinished-task count. Callers subtract online workers
+            when publishing the OpenTelemetry `waiting_tasks` gauge.
+    """
+    cutoff_time = timezone.now() - timedelta(seconds=5)
+
+    incomplete_tasks = (
+        Task.objects.filter(
+            state__in=[TASK_STATES.RUNNING, TASK_STATES.WAITING],
+            pulp_created__lt=cutoff_time,
+        )
+        .order_by("pulp_created")
+        .only("reserved_resources_record")
+    )
+
+    taken_exclusive = set()
+    taken_shared = set()
+    parallel_count = 0
+
+    for task in incomplete_tasks:
+        exclusive_resources, shared_resources = extract_task_resources(task)
+        conflicts = False
+
+        for resource in exclusive_resources:
+            if resource in taken_exclusive or resource in taken_shared:
+                conflicts = True
+                break
+
+        if not conflicts:
+            for resource in shared_resources:
+                if resource in taken_exclusive:
+                    conflicts = True
+                    break
+
+        # Always reserve (even on conflict) so FIFO can't be bypassed; see fetch_task.
+        # e.g. T1(A,B), T2(B,C), T3(C): without reserving C for T2, T3 counts as +1.
+        taken_exclusive.update(exclusive_resources)
+        taken_shared.update(shared_resources)
+        if conflicts:
+            continue
+
+        parallel_count += 1
+
+    return parallel_count
+
+
 class RedisWorker:
     """
     Worker implementation using Redis distributed lock-based resource acquisition.
@@ -133,8 +184,10 @@ class RedisWorker:
         # Metric recording interval
         self.metric_heartbeat_countdown = METRIC_HEARTBEAT_INTERVAL
 
-        # Cache worker count for sleep calculation (updated during beat)
-        self.num_workers = 1
+        # Cache worker count for sleep calculation. Learn the real fleet size at
+        # startup so new workers do not poll at the single-worker rate until the
+        # first heartbeat (~WORKER_TTL/3). Refreshed on each heartbeat in beat().
+        self.num_workers = max(1, AppStatus.objects.online().filter(app_type="worker").count())
 
         # Redis connection for distributed locks
         self.redis_conn = get_redis_connection()
@@ -157,7 +210,10 @@ class RedisWorker:
 
         startup_hook()
 
-        _logger.info("Initialized RedisWorker with Redis lock-based algorithm")
+        _logger.info(
+            "Initialized RedisWorker with Redis lock-based algorithm (online workers=%d)",
+            self.num_workers,
+        )
 
     def _init_instrumentation(self):
         """Initialize OpenTelemetry instrumentation if enabled."""
@@ -210,7 +266,11 @@ class RedisWorker:
             self.app_status.save_heartbeat()
             _logger.debug(msg)
         except (IntegrityError, DatabaseError):
-            _logger.error(f"Updating the heartbeat of worker {self.name} failed.")
+            _logger.error(
+                "Updating the heartbeat of worker %s failed.",
+                self.name,
+                exc_info=True,
+            )
             self.shutdown_requested = True
 
     def handle_redis_heartbeat(self):
@@ -322,19 +382,10 @@ class RedisWorker:
         """
         Record metrics for waiting tasks in the queue.
 
-        This method counts all tasks in RUNNING or WAITING state that are older
-        than 5 seconds, then subtracts the number of active workers to get the
-        number of tasks waiting to be picked up by workers.
+        Publishes `count_waiting_tasks_for_metric() - num_workers` as the
+        OpenTelemetry `waiting_tasks` gauge.
         """
-        cutoff_time = timezone.now() - timedelta(seconds=5)
-
-        task_count = Task.objects.filter(
-            state__in=[TASK_STATES.RUNNING, TASK_STATES.WAITING], pulp_created__lt=cutoff_time
-        ).count()
-
-        waiting_tasks = task_count - self.num_workers
-
-        self.waiting_tasks_meter.set(waiting_tasks)
+        self.waiting_tasks_meter.set(count_waiting_tasks_for_metric() - self.num_workers)
 
     def beat(self):
         """Periodic worker maintenance tasks (heartbeat, cleanup, etc.)."""
@@ -363,7 +414,7 @@ class RedisWorker:
                     self.record_waiting_tasks_metric()
 
             # Update cached worker count for sleep calculation
-            self.num_workers = AppStatus.objects.online().filter(app_type="worker").count()
+            self.num_workers = max(1, AppStatus.objects.online().filter(app_type="worker").count())
 
     def _maybe_release_locks(self, task, mark_released=True):
         """

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore/tests/functional/api/test_crud_domains.py

@@ -360,3 +360,41 @@ def test_set_label_on_domain(pulpcore_bindings, domain_factory, gen_user):
         response = pulpcore_bindings.DomainsApi.unset_label(domain.pulp_href, {"key": "abc"})
         assert response.key == "abc"
         assert response.value == "def"
+
+
+@pytest.mark.parallel
+def test_domain_list_queryset_scoping(pulpcore_bindings, gen_user, gen_object_with_cleanup):
+    """Test that domain list is scoped to domains the user can view."""
+    body = {
+        "name": str(uuid.uuid4()),
+        "storage_class": "pulpcore.app.models.storage.FileSystem",
+        "storage_settings": {"MEDIA_ROOT": ""},
+    }
+    domain = gen_object_with_cleanup(pulpcore_bindings.DomainsApi, body)
+    gen_object_with_cleanup(pulpcore_bindings.DomainsApi, {**body, "name": str(uuid.uuid4())})
+
+    # Authenticated users without view_domain see no domains
+    with gen_user():
+        assert pulpcore_bindings.DomainsApi.list().count == 0
+
+    # Object-level domain_viewer can see only that domain
+    with gen_user(object_roles=[("core.domain_viewer", domain.pulp_href)]):
+        domains = pulpcore_bindings.DomainsApi.list()
+        assert domains.count == 1
+        assert domains.results[0].pulp_href == domain.pulp_href
+
+    # Object-level domain_owner can see only that domain
+    with gen_user(object_roles=[("core.domain_owner", domain.pulp_href)]):
+        domains = pulpcore_bindings.DomainsApi.list()
+        assert domains.count == 1
+        assert domains.results[0].pulp_href == domain.pulp_href
+
+    # Domain creator sees domains they create (via creation_hooks), not others
+    creator = gen_user(model_roles=["core.domain_creator"])
+    with creator:
+        created = gen_object_with_cleanup(
+            pulpcore_bindings.DomainsApi, {**body, "name": str(uuid.uuid4())}
+        )
+        domains = pulpcore_bindings.DomainsApi.list()
+        assert domains.count == 1
+        assert domains.results[0].pulp_href == created.pulp_href

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore/tests/functional/api/using_plugin/test_distributions.py

@@ -167,6 +167,81 @@ def test_distribution_base_path(
     assert json.loads(exc.value.body)["base_path"] is not None
 
 
+@pytest.mark.parallel
+def test_distribution_update_task_reservations(
+    file_bindings,
+    monitor_task,
+):
+    def has_shared_distributions_lock(task):
+        return any(
+            resource.startswith("shared:") and resource.endswith(":distributions")
+            for resource in task.reserved_resources_record
+        )
+
+    def has_exclusive_distributions_lock(task):
+        return any(
+            not resource.startswith("shared:") and resource.endswith(":distributions")
+            for resource in task.reserved_resources_record
+        )
+
+    def has_base_path_lock(task):
+        return any(
+            not resource.startswith("shared:") and resource.endswith(":distribution.base_path")
+            for resource in task.reserved_resources_record
+        )
+
+    create_task = monitor_task(
+        file_bindings.DistributionsFileApi.create(
+            {"name": str(uuid4()), "base_path": str(uuid4())}
+        ).task
+    )
+    assert has_base_path_lock(create_task)
+    assert has_shared_distributions_lock(create_task)
+    assert not has_exclusive_distributions_lock(create_task)
+    distribution = file_bindings.DistributionsFileApi.read(create_task.created_resources[0])
+    assert distribution.prn not in create_task.reserved_resources_record
+
+    no_base_path_update_task = monitor_task(
+        file_bindings.DistributionsFileApi.partial_update(
+            distribution.pulp_href,
+            {"name": str(uuid4())},
+        ).task
+    )
+    assert distribution.prn in no_base_path_update_task.reserved_resources_record
+    assert not has_base_path_lock(no_base_path_update_task)
+    assert has_shared_distributions_lock(no_base_path_update_task)
+    assert not has_exclusive_distributions_lock(no_base_path_update_task)
+
+    unchanged_base_path_update_task = monitor_task(
+        file_bindings.DistributionsFileApi.partial_update(
+            distribution.pulp_href,
+            {"name": str(uuid4()), "base_path": distribution.base_path},
+        ).task
+    )
+    assert distribution.prn in unchanged_base_path_update_task.reserved_resources_record
+    assert not has_base_path_lock(unchanged_base_path_update_task)
+    assert has_shared_distributions_lock(unchanged_base_path_update_task)
+    assert not has_exclusive_distributions_lock(unchanged_base_path_update_task)
+
+    base_path_update_task = monitor_task(
+        file_bindings.DistributionsFileApi.partial_update(
+            distribution.pulp_href, {"base_path": str(uuid4())}
+        ).task
+    )
+    assert distribution.prn in base_path_update_task.reserved_resources_record
+    assert has_base_path_lock(base_path_update_task)
+    assert has_shared_distributions_lock(base_path_update_task)
+    assert not has_exclusive_distributions_lock(base_path_update_task)
+
+    delete_task = monitor_task(
+        file_bindings.DistributionsFileApi.delete(distribution.pulp_href).task
+    )
+    assert distribution.prn in delete_task.reserved_resources_record
+    assert not has_base_path_lock(delete_task)
+    assert has_shared_distributions_lock(delete_task)
+    assert not has_exclusive_distributions_lock(delete_task)
+
+
 @pytest.mark.parallel
 def test_distribution_filtering(
     file_bindings,

pulpcore-3.115.4/pulpcore/tests/unit/tasking/test_waiting_tasks_metric.py

@@ -0,0 +1,132 @@
+from datetime import timedelta
+from uuid import uuid4
+
+import pytest
+from django.utils import timezone
+
+from pulpcore.app.models import Task
+from pulpcore.constants import TASK_STATES
+from pulpcore.tasking.redis_worker import count_waiting_tasks_for_metric
+
+
+@pytest.fixture
+def frozen_clock(monkeypatch):
+    """Move timezone.now() forward so freshly inserted tasks pass the 5s age cutoff.
+
+    Task.pulp_created is set by a DB trigger (clock_timestamp) and cannot be updated,
+    so tests age tasks by advancing the metric's cutoff clock instead.
+    """
+
+    class _Clock:
+        def __init__(self):
+            self.now = timezone.now()
+
+        def __call__(self):
+            return self.now
+
+        def advance(self, seconds):
+            self.now += timedelta(seconds=seconds)
+
+    clock = _Clock()
+    monkeypatch.setattr(timezone, "now", clock)
+    return clock
+
+
+def _create_task(*, state, resources=None, name="test.waiting_tasks_metric"):
+    return Task.objects.create(
+        name=name,
+        state=state,
+        logging_cid="",
+        reserved_resources_record=resources,
+    )
+
+
+@pytest.mark.django_db
+def test_waiting_tasks_metric_resource_contention(frozen_clock):
+    """Exclusive holder + exclusive/shared waiters on same resource → count 1."""
+    exclusive_resource = str(uuid4())
+
+    _create_task(state=TASK_STATES.RUNNING, resources=[exclusive_resource])
+    for _ in range(5):
+        _create_task(state=TASK_STATES.WAITING, resources=[exclusive_resource])
+    for _ in range(5):
+        _create_task(
+            state=TASK_STATES.WAITING,
+            resources=[f"shared:{exclusive_resource}"],
+        )
+
+    frozen_clock.advance(6)
+    assert count_waiting_tasks_for_metric() == 1
+
+
+@pytest.mark.django_db
+def test_waiting_tasks_metric_two_exclusive_lanes(frozen_clock):
+    """Two independent exclusive lanes with waiters → count 2."""
+    resource_a = str(uuid4())
+    resource_b = str(uuid4())
+
+    _create_task(state=TASK_STATES.RUNNING, resources=[resource_a])
+    for _ in range(5):
+        _create_task(state=TASK_STATES.WAITING, resources=[resource_a])
+
+    _create_task(state=TASK_STATES.WAITING, resources=[resource_b])
+    for _ in range(5):
+        _create_task(state=TASK_STATES.WAITING, resources=[resource_b])
+
+    frozen_clock.advance(6)
+    assert count_waiting_tasks_for_metric() == 2
+
+
+@pytest.mark.django_db
+def test_waiting_tasks_metric_shared_resources_can_run_together(frozen_clock):
+    """N shared-only tasks on same resource → count N (not 1)."""
+    shared_resource = str(uuid4())
+    shared_task_count = 5
+
+    for _ in range(shared_task_count):
+        _create_task(
+            state=TASK_STATES.WAITING,
+            resources=[f"shared:{shared_resource}"],
+        )
+
+    frozen_clock.advance(6)
+    assert count_waiting_tasks_for_metric() == shared_task_count
+
+
+@pytest.mark.django_db
+def test_waiting_tasks_metric_fifo_blocks_indirect_resources(frozen_clock):
+    """T1(A,B) running, T2(B,C) and T3(C) waiting → count 1 (FIFO), not 2."""
+    resource_a = str(uuid4())
+    resource_b = str(uuid4())
+    resource_c = str(uuid4())
+
+    # Insert order sets pulp_created via clock_timestamp(); keep create order FIFO.
+    _create_task(state=TASK_STATES.RUNNING, resources=[resource_a, resource_b])
+    _create_task(state=TASK_STATES.WAITING, resources=[resource_b, resource_c])
+    _create_task(state=TASK_STATES.WAITING, resources=[resource_c])
+
+    frozen_clock.advance(6)
+    assert count_waiting_tasks_for_metric() == 1
+
+
+@pytest.mark.django_db
+def test_waiting_tasks_metric_ignores_young_tasks(frozen_clock):
+    """Tasks newer than the five-second cutoff are not counted."""
+    resource = str(uuid4())
+
+    _create_task(state=TASK_STATES.WAITING, resources=[resource])
+    frozen_clock.advance(1)
+
+    assert count_waiting_tasks_for_metric() == 0
+
+
+@pytest.mark.django_db
+def test_waiting_tasks_metric_ignores_finished_tasks(frozen_clock):
+    """Completed/canceled/failed tasks do not contribute to the count."""
+    resource = str(uuid4())
+
+    for state in (TASK_STATES.COMPLETED, TASK_STATES.CANCELED, TASK_STATES.FAILED):
+        _create_task(state=state, resources=[resource])
+
+    frozen_clock.advance(6)
+    assert count_waiting_tasks_for_metric() == 0

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulpcore
-Version: 3.115.2
+Version: 3.115.4
 Summary: Pulp Django Application and Related Modules
 Author-email: Pulp Team <pulp-list@redhat.com>
 License-Expression: GPL-2.0-or-later

{pulpcore-3.115.2 → pulpcore-3.115.4}/pulpcore.egg-info/SOURCES.txt

@@ -499,5 +499,6 @@ pulpcore/tests/unit/stages/__init__.py
 pulpcore/tests/unit/stages/test_artifactdownloader.py
 pulpcore/tests/unit/stages/test_resource_budget.py
 pulpcore/tests/unit/stages/test_stages.py
+pulpcore/tests/unit/tasking/test_waiting_tasks_metric.py
 pulpcore/tests/unit/viewsets/__init__.py
 pulpcore/tests/unit/viewsets/test_viewset_base.py

{pulpcore-3.115.2 → pulpcore-3.115.4}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = "pulpcore"
-version = "3.115.2"
+version = "3.115.4"
 description = "Pulp Django Application and Related Modules"
 readme = "README.md"
 authors = [
@@ -207,7 +207,7 @@ ignore = [
 [tool.bumpversion]
 # This section is managed by the plugin template. Do not edit manually.
 
-current_version = "3.115.2"
+current_version = "3.115.4"
 commit = false
 tag = false
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<alpha>0a)?(?P<patch>\\d+)(\\.(?P<release>[a-z]+))?"
@@ -278,6 +278,7 @@ extend-exclude = [
 
 [tool.ruff.lint]
 # This section is managed by the plugin template. Do not edit manually.
+select = ["E4", "E7", "E9", "F"]
 extend-select = [
   "I",
   "INT",