pulpcore 3.85.0__tar.gz → 3.86.0__tar.gz

{pulpcore-3.85.0 → pulpcore-3.86.0}/CHANGES.md

@@ -8,6 +8,66 @@
 
 [//]: # (towncrier release notes start)
 
+## 3.86.0 (2025-08-21) {: #3.86.0 }
+
+### REST API {: #3.86.0-rest-api }
+
+#### Features {: #3.86.0-rest-api-feature }
+
+- Added a configurable bucket boundary for API request duration metric using the OTEL_PULP_API_HISTOGRAM_BUCKETS setting.
+  [#6845](https://github.com/pulp/pulpcore/issues/6845)
+
+#### Bugfixes {: #3.86.0-rest-api-bugfix }
+
+- Made DjangoMetricsMiddleware aware of OTEL urls exclusion envvars.
+  [#6843](https://github.com/pulp/pulpcore/issues/6843)
+- Fixed repository version validation to allow legitimate shared artifacts between content units.
+  [#6868](https://github.com/pulp/pulpcore/issues/6868)
+- Fixed a bug that prevents migrations from running.
+
+#### Improved Documentation {: #3.86.0-rest-api-doc }
+
+- Updated documentation for the TASK_DIAGNOSTICS setting.
+  [#6854](https://github.com/pulp/pulpcore/issues/6854)
+
+### Plugin API {: #3.86.0-plugin-api }
+
+#### Removals {: #3.86.0-plugin-api-removal }
+
+- Removed support for synchronous immediate tasks.
+
+### Pulp File {: #3.86.0-pulp-file }
+
+No significant changes.
+
+### Pulp Cert Guard {: #3.86.0-pulp-cert-guard }
+
+No significant changes.
+
+---
+
+## 3.85.1 (2025-08-12) {: #3.85.1 }
+
+### REST API {: #3.85.1-rest-api }
+
+#### Bugfixes {: #3.85.1-rest-api-bugfix }
+
+- Fixed a bug that prevents migrations from running.
+
+### Plugin API {: #3.85.1-plugin-api }
+
+No significant changes.
+
+### Pulp File {: #3.85.1-pulp-file }
+
+No significant changes.
+
+### Pulp Cert Guard {: #3.85.1-pulp-cert-guard }
+
+No significant changes.
+
+---
+
 ## 3.85.0 (2025-08-12) {: #3.85.0 }
 
 ### REST API {: #3.85.0-rest-api }

{pulpcore-3.85.0 → pulpcore-3.86.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulpcore
-Version: 3.85.0
+Version: 3.86.0
 Summary: Pulp Django Application and Related Modules
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -45,7 +45,7 @@ Requires-Dist: PyOpenSSL<26.0
 Requires-Dist: opentelemetry-api<1.37,>=1.27.0
 Requires-Dist: opentelemetry-sdk<1.37,>=1.27.0
 Requires-Dist: opentelemetry-exporter-otlp-proto-http<1.37,>=1.27.0
-Requires-Dist: protobuf<6.0,>=4.21.1
+Requires-Dist: protobuf<7.0,>=4.21.1
 Requires-Dist: pulp-glue<0.36,>=0.28.0
 Requires-Dist: pygtrie<=2.5.0,>=2.5
 Requires-Dist: psycopg[binary]<3.3,>=3.1.8

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulp_certguard/app/__init__.py

@@ -6,6 +6,6 @@ class PulpCertGuardPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_certguard.app"
     label = "certguard"
-    version = "3.85.0"
+    version = "3.86.0"
     python_package_name = "pulpcore"
     domain_compatible = True

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulp_file/app/__init__.py

@@ -8,6 +8,6 @@ class PulpFilePluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_file.app"
     label = "file"
-    version = "3.85.0"
+    version = "3.86.0"
     python_package_name = "pulpcore"
     domain_compatible = True

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/app/apps.py

@@ -239,7 +239,7 @@ class PulpAppConfig(PulpPluginAppConfig):
     label = "core"
 
     # The version of this app
-    version = "3.85.0"
+    version = "3.86.0"
 
     # The python package name providing this app
     python_package_name = "pulpcore"

pulpcore-3.86.0/pulpcore/app/management/commands/optimizemigration.py

@@ -0,0 +1,84 @@
+from collections import defaultdict
+from gettext import gettext as _
+
+from django.db import connection
+from django.db.migrations.loader import MigrationLoader
+from django.db.migrations.migration import SwappableTuple
+from django.db.migrations.optimizer import MigrationOptimizer
+from django.db.migrations.writer import MigrationWriter
+from django.conf import settings
+from django.core.management import BaseCommand
+
+
+def print_stats(migration):
+    operations = migration.operations
+    migration_types = defaultdict(int)
+    for operation in operations:
+        migration_types[operation.__class__.__name__] += 1
+    for key, value in migration_types.items():
+        print(f"{value: 4} {key}")
+    print("---")
+    print(_("Total: {count}").format(count=len(operations)))
+
+
+class Command(BaseCommand):
+    """
+    Django management command to optimize a migration.
+    """
+
+    help = _("Optimize a migration.")
+
+    def add_arguments(self, parser):
+        parser.add_argument("--dry-run", action="store_true", help=_("Don't change anything."))
+        parser.add_argument(
+            "--stat", action="store_true", help=_("Print statistics about operations.")
+        )
+        parser.add_argument("app-label", help=_("App label of the migrations to optimize."))
+        parser.add_argument("migration", help=_("Prefix of the migration to optimize."))
+
+    def handle(self, *args, **options):
+        dry_run = options.get("dry_run", False)
+        stat = options.get("stat", False)
+        app_label = options["app-label"]
+        migration_prefix = options["migration"]
+
+        loader = MigrationLoader(connection)
+
+        migration = loader.get_migration_by_prefix(app_label, migration_prefix)
+
+        print(_("Optimizing migration {}").format((migration.app_label, migration.name)))
+        if stat:
+            print(_("=== Old Migration Summary ==="))
+            print_stats(migration)
+
+        new_dependencies = []
+        for dependency in migration.dependencies:
+            if (
+                isinstance(dependency, SwappableTuple)
+                and settings.AUTH_USER_MODEL == dependency.setting
+            ):
+                new_dependencies.append(("__setting__", "AUTH_USER_MODEL"))
+            else:
+                new_dependencies.append(dependency)
+
+        optimizer = MigrationOptimizer()
+        new_operations = optimizer.optimize(migration.operations, app_label)
+
+        if new_operations != migration.operations:
+            print(
+                _("Changed from {old_count} to {new_count} operations.").format(
+                    old_count=len(migration.operations), new_count=len(new_operations)
+                )
+            )
+            if stat:
+                print(_("=== New Migration Summary ==="))
+                print_stats(migration)
+
+            migration.operations = new_operations
+            migration.dependencies = new_dependencies
+            if not dry_run:
+                writer = MigrationWriter(migration)
+                with open(writer.path, "w") as output_file:
+                    output_file.write(writer.as_string())
+        else:
+            print(_("No optimizations found."))

pulpcore-3.86.0/pulpcore/app/migrations/0139_task_app_lock.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.23 on 2025-08-07 11:43
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0138_vulnerabilityreport'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='app_lock',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tasks', to='core.appstatus'),
+        ),
+    ]

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/app/models/status.py

@@ -15,6 +15,7 @@ from pulpcore.app.models import BaseModel
 
 
 class AppStatusManager(models.Manager):
+    # This should be replaced with 3.87.
     def online(self):
         """
         Returns a queryset of objects that are online.
@@ -51,8 +52,15 @@ class AppStatusManager(models.Manager):
 
 class _AppStatusManager(AppStatusManager):
     # This is an intermediate class in order to allow a ZDU.
-    # It should be removed from the chain with 3.87.
+    # It should be made the real thing with 3.87.
+    def __init__(self):
+        super().__init__()
+        self._current_app_status = None
+
     def create(self, app_type, **kwargs):
+        if self._current_app_status is not None:
+            raise RuntimeError("There is already an app status in this process.")
+
         if app_type == "api":
             old_obj = ApiAppStatus.objects.create(**kwargs)
         elif app_type == "worker":
@@ -63,17 +71,27 @@ class _AppStatusManager(AppStatusManager):
             raise NotImplementedError(f"Invalid app_type: {app_type}")
         obj = super().create(app_type=app_type, **kwargs)
         obj._old_status = old_obj
+        self._current_app_status = obj
         return obj
 
     async def acreate(self, app_type, **kwargs):
+        if self._current_app_status is not None:
+            raise RuntimeError("There is already an app status in this process.")
+
         if app_type == "content":
             old_obj = await ContentAppStatus.objects.acreate(**kwargs)
         else:
             raise NotImplementedError(f"Invalid app_type: {app_type}")
         obj = await super().acreate(app_type=app_type, **kwargs)
         obj._old_status = old_obj
+        self._current_app_status = obj
         return obj
 
+    def current(self):
+        if self._current_app_status is None:
+            raise RuntimeError("There is no current app status.")
+        return self._current_app_status
+
     def online(self):
         """
         Returns a queryset of objects that are online.

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/app/models/task.py

@@ -116,7 +116,8 @@ class Task(BaseModel, AutoAddObjPermsMixin):
             Defaults to `True`.
 
     Relations:
-
+        app_lock (AppStatus): The app holding the lock on this task.
+            Warning: This is not yet implemented/enforced.
         parent (models.ForeignKey): Task that spawned this task (if any)
         worker (models.ForeignKey): The worker that this task is in
         pulp_domain (models.ForeignKey): The domain the Task is a part of
@@ -138,6 +139,10 @@ class Task(BaseModel, AutoAddObjPermsMixin):
     enc_kwargs = EncryptedJSONField(null=True, encoder=DjangoJSONEncoder)
 
     worker = models.ForeignKey("Worker", null=True, related_name="tasks", on_delete=models.SET_NULL)
+    # This field is supposed to replace the session advisory locks to protect tasks.
+    app_lock = models.ForeignKey(
+        "AppStatus", null=True, related_name="tasks", on_delete=models.SET_NULL
+    )
 
     parent_task = models.ForeignKey(
         "Task", null=True, related_name="child_tasks", on_delete=models.SET_NULL

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/app/settings.py

@@ -405,6 +405,7 @@ KAFKA_SASL_PASSWORD = None
 
 # opentelemetry settings
 OTEL_ENABLED = False
+OTEL_PULP_API_HISTOGRAM_BUCKETS = []
 
 # VulnerabilityReport settings
 VULN_REPORT_TASK_LIMITER = 10
@@ -504,6 +505,16 @@ authentication_json_header_openapi_security_scheme_validator = Validator(
     messages={"is_type_of": "{name} must be a dictionary."},
 )
 
+otel_pulp_api_histogram_buckets_validator = Validator(
+    "OTEL_PULP_API_HISTOGRAM_BUCKETS",
+    is_type_of=list,
+    condition=lambda v: all([isinstance(value, float) for value in v]),
+    messages={
+        "is_type_of": "{name} must be a list.",
+        "condition": "All buckets must be declared as a float value",
+    },
+)
+
 
 def otel_middleware_hook(settings):
     data = {"dynaconf_merge": True}
@@ -530,6 +541,7 @@ settings = DjangoDynaconf(
         unknown_algs_validator,
         json_header_auth_validator,
         authentication_json_header_openapi_security_scheme_validator,
+        otel_pulp_api_histogram_buckets_validator,
     ],
     post_hooks=(otel_middleware_hook,),
 )

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/middleware.py

@@ -1,5 +1,8 @@
 import time
+import re
+
 from contextvars import ContextVar
+from os import environ
 
 from django.http.response import Http404
 from django.conf import settings
@@ -101,15 +104,32 @@ class APIRootRewriteMiddleware:
 class DjangoMetricsMiddleware:
     def __init__(self, get_response):
         self.meter = init_otel_meter("pulp-api")
-        self.request_duration_histogram = self.meter.create_histogram(
-            name="api.request_duration",
-            description="Tracks the duration of HTTP requests",
-            unit="ms",
-        )
+        self._set_histogram(self.meter)
 
         self.get_response = get_response
 
+    def _excluded_urls(self, url):
+
+        excluded_urls = environ.get(
+            "OTEL_PYTHON_EXCLUDED_URLS", environ.get("OTEL_PYTHON_DJANGO_EXCLUDED_URLS", "")
+        )
+
+        if excluded_urls:
+            excluded_urls_list = [excluded_url.strip() for excluded_url in excluded_urls.split(",")]
+        else:
+            return False
+
+        exclusion_pattern = "|".join(excluded_urls_list)
+
+        if re.search(exclusion_pattern, url):
+            return True
+
+        return False
+
     def __call__(self, request):
+        if self._excluded_urls(request.build_absolute_uri("?")):
+            return self.get_response(request)
+
         start_time = time.time()
         response = self.get_response(request)
         end_time = time.time()
@@ -122,11 +142,18 @@ class DjangoMetricsMiddleware:
         return response
 
     def _set_histogram(self, meter):
-        self.request_duration_histogram = meter.create_histogram(
-            name="api.request_duration",
-            description="Tracks the duration of HTTP requests",
-            unit="ms",
-        )
+        create_histogram_kwargs = {
+            "name": "api.request_duration",
+            "description": "Tracks the duration of HTTP requests",
+            "unit": "ms",
+        }
+
+        if settings.OTEL_PULP_API_HISTOGRAM_BUCKETS:
+            create_histogram_kwargs["explicit_bucket_boundaries_advisory"] = (
+                settings.OTEL_PULP_API_HISTOGRAM_BUCKETS
+            )
+
+        self.request_duration_histogram = meter.create_histogram(**create_histogram_kwargs)
 
     def _process_attributes(self, request, response):
         return {

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/migrations.py

@@ -3,8 +3,6 @@ from packaging.version import parse as parse_version
 from django.conf import settings
 from django.utils import timezone
 from django.db.migrations.operations.base import Operation
-from django.db.models import F
-from django.db.models.functions import Now
 
 
 class RequireVersion(Operation):
@@ -58,7 +56,10 @@ class RequireVersion(Operation):
 
         try:
             AppStatus = from_state.apps.get_model("core", "AppStatus")
-            for worker in AppStatus.objects.filter(F("last_heartbeat") + F("ttl") >= Now()):
+        except LookupError:
+            pass
+        else:
+            for worker in AppStatus.objects.all():
                 present_version = worker.versions.get(self.plugin)
                 if present_version is not None and parse_version(present_version) < needed_version:
                     errors.append(
@@ -67,8 +68,6 @@ class RequireVersion(Operation):
                     )
 
             found_either_table = True
-        except LookupError:
-            pass
 
         assert found_either_table
         if errors:

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/plugin/repo_version_utils.py

@@ -116,9 +116,15 @@ def validate_version_paths(version):
     Raises:
         ValueError: If two artifact relative paths overlap
     """
-    paths = ContentArtifact.objects.filter(content__pk__in=version.content).values_list(
-        "relative_path", flat=True
+    # Get unique (path, artifact) pairs to allow artifacts shared across content
+    content_artifacts = (
+        ContentArtifact.objects.filter(content__pk__in=version.content)
+        .values_list("relative_path", "artifact")
+        .distinct()
     )
+
+    paths = [path for path, artifact_id in content_artifacts]
+
     try:
         validate_file_paths(paths)
     except ValueError as e:

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/pytest_plugin.py

@@ -1013,23 +1013,25 @@ def dispatch_task(pulpcore_bindings):
         commands = (
             "from django_guid import set_guid; "
             "from pulpcore.tasking.tasks import dispatch; "
-            "from pulpcore.app.models import TaskGroup; "
+            "from pulpcore.app.models import TaskGroup, AppStatus; "
             "from pulpcore.app.util import get_url, set_current_user; "
             "from django.contrib.auth import get_user_model; "
+            f"app_status=AppStatus.objects.create(name='test-' + {cid!r},app_type='worker'); "
             "User = get_user_model(); "
             f"user = User.objects.filter(username='{username}').first(); "
             "set_current_user(user); "
             f"set_guid({cid!r}); "
             f"tg = {task_group_id!r} and TaskGroup.objects.filter(pk={task_group_id!r}).first(); "
             f"task = dispatch(*{args!r}, task_group=tg, **{kwargs!r}); "
+            "app_status.delete(); "
             "print(get_url(task))"
         )
 
         process = subprocess.run(["pulpcore-manager", "shell", "-c", commands], capture_output=True)
-
-        assert process.returncode == 0
+        err_log = process.stderr.decode()
+        assert process.returncode == 0, err_log
         task_href = process.stdout.decode().strip()
-        print(process.stderr.decode(), file=sys.stderr)
+        print(err_log, file=sys.stderr)
         return task_href
 
     return _dispatch_task

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/tasking/tasks.py

@@ -8,7 +8,6 @@ import sys
 import traceback
 import tempfile
 import threading
-from asgiref.sync import sync_to_async
 from gettext import gettext as _
 
 from django.conf import settings
@@ -16,8 +15,12 @@ from django.db import connection, transaction
 from django.db.models import Model
 from django_guid import get_guid
 from pulpcore.app.apps import MODULE_PLUGIN_VERSIONS
-from pulpcore.app.models import Task, TaskGroup
-from pulpcore.app.util import current_task, get_domain, get_prn, deprecation_logger
+from pulpcore.app.models import Task, TaskGroup, AppStatus
+from pulpcore.app.util import (
+    current_task,
+    get_domain,
+    get_prn,
+)
 from pulpcore.constants import (
     TASK_FINAL_STATES,
     TASK_INCOMPLETE_STATES,
@@ -81,19 +84,11 @@ def _execute_task(task):
         immediate = task.immediate
         is_coroutine_fn = asyncio.iscoroutinefunction(func)
 
-        if not is_coroutine_fn:
-            if immediate:
-                deprecation_logger.warning(
-                    "Immediate tasks must be coroutine functions. "
-                    "Support for non-coroutine immediate tasks will be dropped "
-                    "in pulpcore 3.85."
-                )
-                func = sync_to_async(func)
-                is_coroutine_fn = True
-            else:
-                func(*args, **kwargs)
+        if immediate and not is_coroutine_fn:
+            raise ValueError("Immediate tasks must be async functions.")
 
         if is_coroutine_fn:
+            # both regular and immediate tasks can be coroutines, but only immediate must timeout
             _logger.debug("Task is coroutine %s", task.pk)
             coro = func(*args, **kwargs)
             if immediate:
@@ -110,6 +105,8 @@ def _execute_task(task):
                         timeout=IMMEDIATE_TIMEOUT,
                     )
                 )
+        else:
+            func(*args, **kwargs)
 
     except Exception:
         exc_type, exc, tb = sys.exc_info()
@@ -240,6 +237,7 @@ def dispatch(
                 immediate=immediate,
                 deferred=deferred,
                 profile_options=x_task_diagnostics_var.get(None),
+                app_lock=(immediate and AppStatus.objects.current()) or None,
             )
             task.refresh_from_db()  # The database may have assigned a timestamp for us.
             if immediate:
@@ -272,13 +270,16 @@ def dispatch(
                     try:
                         execute_task(task)
                     finally:
-                        # whether the task fails or not, we should always restore the workdir
+                        # Whether the task fails or not, we should always restore the workdir.
                         os.chdir(cur_dir)
 
                 if resources:
                     notify_workers = True
             elif deferred:
+                # Resources are blocked. Let the others handle it.
                 notify_workers = True
+                task.app_lock = None
+                task.save()
             else:
                 task.set_canceling()
                 task.set_canceled(TASK_STATES.CANCELED, "Resources temporarily unavailable.")

{pulpcore-3.85.0 → pulpcore-3.86.0}/pulpcore/tasking/worker.py

@@ -363,41 +363,60 @@ class PulpcoreWorker:
     def iter_tasks(self):
         """Iterate over ready tasks and yield each task while holding the lock."""
         while not self.shutdown_requested:
-            # When batching this query, be sure to use "pulp_created" as a cursor
+            # When batching this query, be sure to use "pulp_created" as a cursor.
             for task in Task.objects.filter(
                 state__in=TASK_INCOMPLETE_STATES,
                 unblocked_at__isnull=False,
             ).order_by("-immediate", F("pulp_created") + Value(timedelta(seconds=8)) * Random()):
-                # This code will only be called if we acquired the lock successfully
-                # The lock will be automatically be released at the end of the block
+                # This code will only be called if we acquired the lock successfully.
+                # The lock will be automatically be released at the end of the block.
                 with contextlib.suppress(AdvisoryLockError), task:
-                    # Check if someone else changed the task before we got the lock
-                    task.refresh_from_db()
-
-                    if task.state == TASK_STATES.CANCELING and task.worker is None:
-                        # No worker picked this task up before being canceled
-                        if self.cancel_abandoned_task(task, TASK_STATES.CANCELED):
-                            # Continue looking for the next task without considering this
-                            # tasks resources, as we just released them
-                            continue
-                    if task.state in [TASK_STATES.RUNNING, TASK_STATES.CANCELING]:
-                        # A running task without a lock must be abandoned
-                        if self.cancel_abandoned_task(
-                            task, TASK_STATES.FAILED, "Worker has gone missing."
+                    # We got the advisory lock (OLD) now try to get the app_lock (NEW).
+                    rows = Task.objects.filter(pk=task.pk, app_lock=None).update(
+                        app_lock=AppStatus.objects.current()
+                    )
+                    if rows == 0:
+                        _logger.error(
+                            "Acquired advisory lock but missed the app_lock for the task. "
+                            "This should only happen during the upgrade phase to the new app_lock."
+                        )
+                        continue
+                    try:
+                        # Check if someone else changed the task before we got the lock.
+                        task.refresh_from_db()
+
+                        if task.state == TASK_STATES.CANCELING and task.worker is None:
+                            # No worker picked this task up before being canceled.
+                            if self.cancel_abandoned_task(task, TASK_STATES.CANCELED):
+                                # Continue looking for the next task without considering this
+                                # tasks resources, as we just released them.
+                                continue
+                        if task.state in [TASK_STATES.RUNNING, TASK_STATES.CANCELING]:
+                            # A running task without a lock must be abandoned.
+                            if self.cancel_abandoned_task(
+                                task, TASK_STATES.FAILED, "Worker has gone missing."
+                            ):
+                                # Continue looking for the next task without considering this
+                                # tasks resources, as we just released them.
+                                continue
+
+                        # This statement is using lazy evaluation.
+                        if (
+                            task.state == TASK_STATES.WAITING
+                            and task.unblocked_at is not None
+                            and self.is_compatible(task)
                         ):
-                            # Continue looking for the next task without considering this
-                            # tasks resources, as we just released them
-                            continue
-
-                    # This statement is using lazy evaluation
-                    if (
-                        task.state == TASK_STATES.WAITING
-                        and task.unblocked_at is not None
-                        and self.is_compatible(task)
-                    ):
-                        yield task
-                        # Start from the top of the Task list
-                        break
+                            yield task
+                            # Start from the top of the Task list.
+                            break
+                    finally:
+                        rows = Task.objects.filter(
+                            pk=task.pk, app_lock=AppStatus.objects.current()
+                        ).update(app_lock=None)
+                        if rows != 1:
+                            raise RuntimeError(
+                                "Something other than us is messing around with locks."
+                            )
             else:
                 # No task found in the for-loop
                 break