pulpcore 3.85.1__py3-none-any.whl → 3.87.0__py3-none-any.whl

pulpcore/plugin/repo_version_utils.py

@@ -116,9 +116,15 @@ def validate_version_paths(version):
     Raises:
         ValueError: If two artifact relative paths overlap
     """
-    paths = ContentArtifact.objects.filter(content__pk__in=version.content).values_list(
-        "relative_path", flat=True
+    # Get unique (path, artifact) pairs to allow artifacts shared across content
+    content_artifacts = (
+        ContentArtifact.objects.filter(content__pk__in=version.content)
+        .values_list("relative_path", "artifact")
+        .distinct()
     )
+
+    paths = [path for path, artifact_id in content_artifacts]
+
     try:
         validate_file_paths(paths)
     except ValueError as e:

pulpcore/pytest_plugin.py

@@ -1013,23 +1013,25 @@ def dispatch_task(pulpcore_bindings):
         commands = (
             "from django_guid import set_guid; "
             "from pulpcore.tasking.tasks import dispatch; "
-            "from pulpcore.app.models import TaskGroup; "
+            "from pulpcore.app.models import TaskGroup, AppStatus; "
             "from pulpcore.app.util import get_url, set_current_user; "
             "from django.contrib.auth import get_user_model; "
+            f"app_status=AppStatus.objects.create(name='test-' + {cid!r},app_type='worker'); "
             "User = get_user_model(); "
             f"user = User.objects.filter(username='{username}').first(); "
             "set_current_user(user); "
             f"set_guid({cid!r}); "
             f"tg = {task_group_id!r} and TaskGroup.objects.filter(pk={task_group_id!r}).first(); "
             f"task = dispatch(*{args!r}, task_group=tg, **{kwargs!r}); "
+            "app_status.delete(); "
             "print(get_url(task))"
         )
 
         process = subprocess.run(["pulpcore-manager", "shell", "-c", commands], capture_output=True)
-
-        assert process.returncode == 0
+        err_log = process.stderr.decode()
+        assert process.returncode == 0, err_log
         task_href = process.stdout.decode().strip()
-        print(process.stderr.decode(), file=sys.stderr)
+        print(err_log, file=sys.stderr)
         return task_href
 
     return _dispatch_task

pulpcore/tasking/tasks.py

@@ -8,7 +8,6 @@ import sys
 import traceback
 import tempfile
 import threading
-from asgiref.sync import sync_to_async
 from gettext import gettext as _
 
 from django.conf import settings
@@ -16,8 +15,12 @@ from django.db import connection, transaction
 from django.db.models import Model
 from django_guid import get_guid
 from pulpcore.app.apps import MODULE_PLUGIN_VERSIONS
-from pulpcore.app.models import Task, TaskGroup
-from pulpcore.app.util import current_task, get_domain, get_prn, deprecation_logger
+from pulpcore.app.models import Task, TaskGroup, AppStatus
+from pulpcore.app.util import (
+    current_task,
+    get_domain,
+    get_prn,
+)
 from pulpcore.constants import (
     TASK_FINAL_STATES,
     TASK_INCOMPLETE_STATES,
@@ -81,19 +84,11 @@ def _execute_task(task):
         immediate = task.immediate
         is_coroutine_fn = asyncio.iscoroutinefunction(func)
 
-        if not is_coroutine_fn:
-            if immediate:
-                deprecation_logger.warning(
-                    "Immediate tasks must be coroutine functions. "
-                    "Support for non-coroutine immediate tasks will be dropped "
-                    "in pulpcore 3.85."
-                )
-                func = sync_to_async(func)
-                is_coroutine_fn = True
-            else:
-                func(*args, **kwargs)
+        if immediate and not is_coroutine_fn:
+            raise ValueError("Immediate tasks must be async functions.")
 
         if is_coroutine_fn:
+            # both regular and immediate tasks can be coroutines, but only immediate must timeout
             _logger.debug("Task is coroutine %s", task.pk)
             coro = func(*args, **kwargs)
             if immediate:
@@ -110,6 +105,8 @@ def _execute_task(task):
                         timeout=IMMEDIATE_TIMEOUT,
                     )
                 )
+        else:
+            func(*args, **kwargs)
 
     except Exception:
         exc_type, exc, tb = sys.exc_info()
@@ -240,6 +237,7 @@ def dispatch(
                 immediate=immediate,
                 deferred=deferred,
                 profile_options=x_task_diagnostics_var.get(None),
+                app_lock=(immediate and AppStatus.objects.current()) or None,
             )
             task.refresh_from_db()  # The database may have assigned a timestamp for us.
             if immediate:
@@ -272,13 +270,16 @@ def dispatch(
                     try:
                         execute_task(task)
                     finally:
-                        # whether the task fails or not, we should always restore the workdir
+                        # Whether the task fails or not, we should always restore the workdir.
                         os.chdir(cur_dir)
 
                 if resources:
                     notify_workers = True
             elif deferred:
+                # Resources are blocked. Let the others handle it.
                 notify_workers = True
+                task.app_lock = None
+                task.save()
             else:
                 task.set_canceling()
                 task.set_canceled(TASK_STATES.CANCELED, "Resources temporarily unavailable.")

pulpcore/tasking/worker.py

@@ -77,7 +77,6 @@ class PulpcoreWorker:
             self.app_status = AppStatus.objects.create(
                 name=self.name, app_type="worker", versions=self.versions
             )
-            self.worker = self.app_status._old_status
         except IntegrityError:
             _logger.error(f"A worker with name {self.name} already exists in the database.")
             exit(1)
@@ -184,15 +183,17 @@ class PulpcoreWorker:
         for app_worker in qs:
             _logger.info(_("Clean missing %s worker %s."), app_worker.app_type, app_worker.name)
         qs.delete()
-        for cls, cls_name in (
-            (Worker, "pulp"),
-            (ApiAppStatus, "api"),
-            (ContentAppStatus, "content"),
-        ):
-            qs = cls.objects.missing(age=timedelta(days=7))
-            for app_worker in qs:
-                _logger.info(_("Clean missing %s worker %s."), cls_name, app_worker.name)
-            qs.delete()
+        with contextlib.suppress(DatabaseError):
+            # By now a migration on a newer release may have deleted these tables already.
+            for cls, cls_name in (
+                (Worker, "pulp"),
+                (ApiAppStatus, "api"),
+                (ContentAppStatus, "content"),
+            ):
+                qs = cls.objects.missing(age=timedelta(days=7))
+                for app_worker in qs:
+                    _logger.info(_("Clean missing %s worker %s."), cls_name, app_worker.name)
+                qs.delete()
 
     def beat(self):
         if self.app_status.last_heartbeat < timezone.now() - self.heartbeat_period:
@@ -363,41 +364,60 @@ class PulpcoreWorker:
     def iter_tasks(self):
         """Iterate over ready tasks and yield each task while holding the lock."""
         while not self.shutdown_requested:
-            # When batching this query, be sure to use "pulp_created" as a cursor
+            # When batching this query, be sure to use "pulp_created" as a cursor.
             for task in Task.objects.filter(
                 state__in=TASK_INCOMPLETE_STATES,
                 unblocked_at__isnull=False,
             ).order_by("-immediate", F("pulp_created") + Value(timedelta(seconds=8)) * Random()):
-                # This code will only be called if we acquired the lock successfully
-                # The lock will be automatically be released at the end of the block
+                # This code will only be called if we acquired the lock successfully.
+                # The lock will be automatically be released at the end of the block.
                 with contextlib.suppress(AdvisoryLockError), task:
-                    # Check if someone else changed the task before we got the lock
-                    task.refresh_from_db()
-
-                    if task.state == TASK_STATES.CANCELING and task.worker is None:
-                        # No worker picked this task up before being canceled
-                        if self.cancel_abandoned_task(task, TASK_STATES.CANCELED):
-                            # Continue looking for the next task without considering this
-                            # tasks resources, as we just released them
-                            continue
-                    if task.state in [TASK_STATES.RUNNING, TASK_STATES.CANCELING]:
-                        # A running task without a lock must be abandoned
-                        if self.cancel_abandoned_task(
-                            task, TASK_STATES.FAILED, "Worker has gone missing."
+                    # We got the advisory lock (OLD) now try to get the app_lock (NEW).
+                    rows = Task.objects.filter(pk=task.pk, app_lock=None).update(
+                        app_lock=AppStatus.objects.current()
+                    )
+                    if rows == 0:
+                        _logger.error(
+                            "Acquired advisory lock but missed the app_lock for the task. "
+                            "This should only happen during the upgrade phase to the new app_lock."
+                        )
+                        continue
+                    try:
+                        # Check if someone else changed the task before we got the lock.
+                        task.refresh_from_db()
+
+                        if task.state == TASK_STATES.CANCELING:
+                            # No worker picked this task up before being canceled.
+                            if self.cancel_abandoned_task(task, TASK_STATES.CANCELED):
+                                # Continue looking for the next task without considering this
+                                # tasks resources, as we just released them.
+                                continue
+                        if task.state == TASK_STATES.RUNNING:
+                            # A running task without a lock must be abandoned.
+                            if self.cancel_abandoned_task(
+                                task, TASK_STATES.FAILED, "Worker has gone missing."
+                            ):
+                                # Continue looking for the next task without considering this
+                                # tasks resources, as we just released them.
+                                continue
+
+                        # This statement is using lazy evaluation.
+                        if (
+                            task.state == TASK_STATES.WAITING
+                            and task.unblocked_at is not None
+                            and self.is_compatible(task)
                         ):
-                            # Continue looking for the next task without considering this
-                            # tasks resources, as we just released them
-                            continue
-
-                    # This statement is using lazy evaluation
-                    if (
-                        task.state == TASK_STATES.WAITING
-                        and task.unblocked_at is not None
-                        and self.is_compatible(task)
-                    ):
-                        yield task
-                        # Start from the top of the Task list
-                        break
+                            yield task
+                            # Start from the top of the Task list.
+                            break
+                    finally:
+                        rows = Task.objects.filter(
+                            pk=task.pk, app_lock=AppStatus.objects.current()
+                        ).update(app_lock=None)
+                        if rows != 1:
+                            raise RuntimeError(
+                                "Something other than us is messing around with locks."
+                            )
             else:
                 # No task found in the for-loop
                 break
@@ -426,8 +446,6 @@ class PulpcoreWorker:
 
         self.cancel_task = False
         self.task = task
-        task.worker = self.worker
-        task.save(update_fields=["worker"])
         cancel_state = None
         cancel_reason = None
         domain = task.pulp_domain

pulpcore/tests/functional/api/test_tasking.py

@@ -230,25 +230,6 @@ def test_retrieve_task_with_minimal_fields(task, bindings_cfg):
     assert unexpected_fields.isdisjoint(returned_fields)
 
 
-@pytest.mark.parallel
-def test_retrieve_task_using_invalid_worker(pulpcore_bindings):
-    """Expects to raise an exception when using invalid worker value as filter."""
-
-    with pytest.raises(ApiException) as ctx:
-        pulpcore_bindings.TasksApi.list(worker=str(uuid4()))
-
-    assert ctx.value.status == 400
-
-
-@pytest.mark.parallel
-def test_retrieve_task_using_valid_worker(task, pulpcore_bindings):
-    """Expects to retrieve a task using a valid worker URI as filter."""
-
-    response = pulpcore_bindings.TasksApi.list(worker=task.worker)
-
-    assert response.results and response.count
-
-
 @pytest.mark.parallel
 def test_retrieve_task_using_valid_date(task, pulpcore_bindings):
     """Expects to retrieve a task using a valid date."""
@@ -278,22 +259,6 @@ def test_search_task_using_an_invalid_name(pulpcore_bindings):
     assert not search_results.results and not search_results.count
 
 
-@pytest.mark.parallel
-def test_filter_tasks_using_worker__in_filter(pulpcore_bindings, dispatch_task, monitor_task):
-    task1_href = dispatch_task("pulpcore.app.tasks.test.sleep", args=(0,))
-    task2_href = dispatch_task("pulpcore.app.tasks.test.sleep", args=(0,))
-
-    task1 = monitor_task(task1_href)
-    task2 = monitor_task(task2_href)
-
-    search_results = pulpcore_bindings.TasksApi.list(worker__in=(task1.worker, task2.worker))
-
-    tasks_hrefs = [task.pulp_href for task in search_results.results]
-
-    assert task1_href in tasks_hrefs
-    assert task2_href in tasks_hrefs
-
-
 @pytest.mark.parallel
 def test_filter_tasks_using_pulp_created_filter(pulpcore_bindings, dispatch_task):
 
@@ -487,25 +452,23 @@ class TestImmediateTaskWithNoResource:
         )
         task = pulpcore_bindings.TasksApi.read(task_href)
         assert task.state == "completed"
-        assert task.worker is None
 
     @pytest.mark.parallel
-    def test_executes_on_api_worker_when_no_async(self, pulpcore_bindings, dispatch_task, capsys):
+    def test_executes_on_api_worker_when_no_async(
+        self, pulpcore_bindings, dispatch_task, monitor_task
+    ):
         """
         GIVEN a task with no resource requirements
         AND the task IS NOT an async function
         WHEN dispatching a task as immediate
-        THEN the task completes with no associated worker
+        THEN the dispatch should throw an error
         """
-        # TODO: on 3.85 this should throw an error
-        task_href = dispatch_task(
-            "pulpcore.app.tasks.test.sleep", args=(LT_TIMEOUT,), immediate=True
-        )
-        stderr_content = capsys.readouterr().err
-        task = pulpcore_bindings.TasksApi.read(task_href)
-        assert task.state == "completed"
-        assert task.worker is None
-        assert "Support for non-coroutine immediate tasks will be dropped" in stderr_content
+        with pytest.raises(PulpTaskError) as ctx:
+            task_href = dispatch_task(
+                "pulpcore.app.tasks.test.sleep", args=(LT_TIMEOUT,), immediate=True
+            )
+            monitor_task(task_href)
+        assert "Immediate tasks must be async functions" in ctx.value.task.error["description"]
 
     @pytest.mark.parallel
     def test_timeouts_on_api_worker(self, pulpcore_bindings, dispatch_task):
@@ -520,7 +483,7 @@ class TestImmediateTaskWithNoResource:
             "pulpcore.app.tasks.test.asleep", args=(GT_TIMEOUT,), immediate=True
         )
         task = pulpcore_bindings.TasksApi.read(task_href)
-        assert task.worker is None
+        assert task.state == "failed"
         assert "task timed out after" in task.error["description"]
 
 
@@ -556,7 +519,7 @@ class TestImmediateTaskWithBlockedResource:
         """
         GIVEN an async task requiring busy resources
         WHEN dispatching a task as immediate
-        THEN the task completes with a worker
+        THEN the task returns as waiting first
         """
         COMMON_RESOURCE = str(uuid4())
         with resource_blocker(exclusive_resources=[COMMON_RESOURCE]):
@@ -566,9 +529,10 @@ class TestImmediateTaskWithBlockedResource:
                 immediate=True,
                 exclusive_resources=[COMMON_RESOURCE],
             )
+            task = pulpcore_bindings.TasksApi.read(task_href)
+            assert task.state == "waiting"
         task = monitor_task(task_href)
         assert task.state == "completed"
-        assert task.worker is not None
 
     @pytest.mark.parallel
     def test_throws_when_non_deferrable(
@@ -590,7 +554,6 @@ class TestImmediateTaskWithBlockedResource:
             )
             task = pulpcore_bindings.TasksApi.read(task_href)
             assert task.state == "canceled"
-            assert task.worker is None
             assert "Resources temporarily unavailable." in task.error["reason"]
 
     @pytest.mark.parallel

pulpcore/tests/functional/api/test_workers.py

@@ -4,7 +4,6 @@ import pytest
 import subprocess
 import uuid
 from datetime import datetime, timedelta
-from random import choice
 from time import sleep
 
 
@@ -23,7 +22,7 @@ def test_worker_actions(pulpcore_bindings):
             assert val is not None
 
     # Pick a random worker to be used for the next assertions.
-    chosen_worker = choice(workers)
+    chosen_worker = next(worker for worker in workers if not worker.name.startswith("test-"))
 
     # Read a worker by its pulp_href.
     read_worker = pulpcore_bindings.WorkersApi.read(chosen_worker.pulp_href)

pulpcore/tests/unit/content/test_heartbeat.py

@@ -5,7 +5,7 @@ from django.db.utils import InterfaceError, OperationalError
 
 from pulpcore.content import _heartbeat
 from pulpcore.content.handler import Handler
-from pulpcore.app.models.status import AppStatusManager
+from pulpcore.app.models.status import AppStatus, AppStatusManager
 
 
 class MockException(Exception):
@@ -25,6 +25,7 @@ async def test_db_connection_interface_error(monkeypatch, settings, error_class)
     mock_acreate = AsyncMock()
     mock_acreate.return_value = mock_app_status
     monkeypatch.setattr(AppStatusManager, "acreate", mock_acreate)
+    monkeypatch.setattr(AppStatus, "objects", AppStatusManager())
     mock_reset_db = Mock()
     monkeypatch.setattr(Handler, "_reset_db_connection", mock_reset_db)
     settings.CONTENT_APP_TTL = 1

pulpcore/tests/unit/models/test_repository.py

@@ -3,7 +3,8 @@ from uuid import uuid4
 
 from itertools import compress
 
-from pulpcore.plugin.models import Content, Repository
+from pulpcore.plugin.models import Artifact, Content, ContentArtifact, Repository
+from pulpcore.plugin.repo_version_utils import validate_version_paths
 
 
 def pks_of_next_qs(qs_generator):
@@ -258,3 +259,85 @@ def test_next_version_with_multiple_versions():
 
     assert repository.next_version == 4
     assert repository.latest_version().number == 1
+
+
+@pytest.mark.django_db
+def test_shared_artifact_same_path_validation(tmp_path):
+    """
+    Test that multiple content units can reference the same artifact with the same
+    relative path without causing validation errors.
+
+    This reproduces scenarios where different content units legitimately share
+    the same artifact (e.g. upstream source files).
+    """
+    # Create a repository
+    repository = Repository.objects.create(name=uuid4())
+    repository.CONTENT_TYPES = [Content]
+
+    # Create a shared artifact using proper test pattern
+    artifact_path = tmp_path / "shared_file.txt"
+    artifact_path.write_text("Shared content data")
+    shared_artifact = Artifact.init_and_validate(str(artifact_path))
+    shared_artifact.save()
+
+    # Create two content units (simulates any content that shares artifacts)
+    content1 = Content.objects.create(pulp_type="core.content")
+    content2 = Content.objects.create(pulp_type="core.content")
+
+    # Both content units reference the same artifact with same path
+    ContentArtifact.objects.create(
+        content=content1, artifact=shared_artifact, relative_path="shared/common_file.txt"
+    )
+    ContentArtifact.objects.create(
+        content=content2, artifact=shared_artifact, relative_path="shared/common_file.txt"
+    )
+
+    # Create a repository version with both content units
+    with repository.new_version() as new_version:
+        new_version.add_content(Content.objects.filter(pk__in=[content1.pk, content2.pk]))
+
+    # This should not raise validation errors with our fix
+    validate_version_paths(new_version)
+
+
+@pytest.mark.django_db
+def test_different_artifacts_same_path_validation_fails(tmp_path):
+    """
+    Test that different artifacts trying to use the same relative path
+    still fail validation (this is a real conflict that should be caught).
+    """
+    # Create a repository
+    repository = Repository.objects.create(name=uuid4())
+    repository.CONTENT_TYPES = [Content]
+
+    # Create two different artifacts using proper test pattern
+    artifact1_path = tmp_path / "artifact1.txt"
+    artifact1_path.write_text("Content of first artifact")
+    artifact1 = Artifact.init_and_validate(str(artifact1_path))
+    artifact1.save()
+
+    artifact2_path = tmp_path / "artifact2.txt"
+    artifact2_path.write_text("Content of second artifact")  # Different content
+    artifact2 = Artifact.init_and_validate(str(artifact2_path))
+    artifact2.save()
+
+    # Create two content units with different artifacts but same path
+    content1 = Content.objects.create(pulp_type="core.content")
+    content2 = Content.objects.create(pulp_type="core.content")
+
+    ContentArtifact.objects.create(
+        content=content1, artifact=artifact1, relative_path="conflicting/file.txt"
+    )
+    ContentArtifact.objects.create(
+        content=content2,
+        artifact=artifact2,
+        relative_path="conflicting/file.txt",  # Same path, different artifact
+    )
+
+    # Create a repository version with both content units
+    with repository.new_version() as new_version:
+        new_version.add_content(Content.objects.filter(pk__in=[content1.pk, content2.pk]))
+
+    # This should raise a validation error due to path conflict
+    with pytest.raises(ValueError, match="Repository version errors"):
+        validate_version_paths(new_version)

{pulpcore-3.85.1.dist-info → pulpcore-3.87.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulpcore
-Version: 3.85.1
+Version: 3.87.0
 Summary: Pulp Django Application and Related Modules
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -45,12 +45,12 @@ Requires-Dist: PyOpenSSL<26.0
 Requires-Dist: opentelemetry-api<1.37,>=1.27.0
 Requires-Dist: opentelemetry-sdk<1.37,>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp-proto-http<1.37,>=1.27.0
-Requires-Dist: protobuf<6.0,>=4.21.1
+Requires-Dist: protobuf<7.0,>=4.21.1
 Requires-Dist: pulp-glue<0.36,>=0.28.0
 Requires-Dist: pygtrie<=2.5.0,>=2.5
 Requires-Dist: psycopg[binary]<3.3,>=3.1.8
 Requires-Dist: pyparsing<3.3,>=3.1.0
-Requires-Dist: python-gnupg<=0.5.4,>=0.5
+Requires-Dist: python-gnupg<0.6,>=0.5.0
 Requires-Dist: PyYAML<6.1,>=5.1.1
 Requires-Dist: redis<6.5,>=4.3.0
 Requires-Dist: tablib<3.6,>=3.5.0