pulpcore 3.83.2__py3-none-any.whl → 3.85.0__py3-none-any.whl

pulpcore/app/tasks/vulnerability_report.py

@@ -0,0 +1,159 @@
+import aiohttp
+import asyncio
+import importlib
+
+from asgiref.sync import sync_to_async
+from django.conf import settings
+
+from pulpcore.app.util import get_domain
+from pulpcore.app.models.progress import ProgressReport
+from pulpcore.app.models.vulnerability_report import VulnerabilityReport
+from pulpcore.constants import OSV_QUERY_URL, TASK_STATES
+
+
+class VulnerabilityReportScanner:
+    """
+    A scanner class to collect vulnerabilities from Pulp Contents using the OSV.dev API.
+
+    Attributes:
+        semaphore (asyncio.Semaphore): Controls concurrent API requests to osv.dev service.
+        generator (AsyncGenerator): The async generator with the content data to be scanned for
+            vulnerabilities.
+        session (aiohttp.ClientSession): HTTP session for making requests to the OSV.dev API.
+            Defaults to a new session if not provided.
+        created (ProgressReport): Tracks the number of new VulnerabilityReport records created
+            during the scanning process.
+        updated (ProgressReport): Tracks the number of existing VulnerabilityReport records
+            updated during the scanning process.
+        total_scanned (ProgressReport): Tracks the total number of content items that have
+            been processed for vulnerability scanning.
+    """
+
+    def __init__(self, semaphore, generator, session=None):
+        self.semaphore = semaphore
+        self.generator = generator
+        self.session = session or aiohttp.ClientSession()
+        self.created = ProgressReport(
+            message="Vulnerability Reports created", code="created", state=TASK_STATES.RUNNING
+        )
+        self.updated = ProgressReport(
+            message="Vulnerability Reports updated", code="updated", state=TASK_STATES.RUNNING
+        )
+        self.total_scanned = ProgressReport(
+            message="Content scanned", code="total_scanned", state=TASK_STATES.RUNNING
+        )
+
+    async def scan_packages(self):
+        """
+        Main entry point for scanning packages for vulnerabilities. Progress is tracked through
+        three ProgressReport instances (created, updated, total_scanned).
+        """
+        tasks = []
+        async for content in self.generator:
+            tasks.append(asyncio.create_task(self.scan_package(content)))
+        await asyncio.gather(*tasks)
+        await self.session.close()
+
+        if self.created.done > 0:
+            self.created.state = TASK_STATES.COMPLETED
+            await self.created.asave()
+        if self.total_scanned.done > 0:
+            self.total_scanned.state = TASK_STATES.COMPLETED
+            await self.total_scanned.asave()
+        if self.updated.done > 0:
+            self.updated.state = TASK_STATES.COMPLETED
+            await self.updated.asave()
+
+    async def scan_package(self, repo_content_osv_data):
+        """
+        Makes a request to the osv.dev API and store the results in VulnerabilityReport model.
+
+        Args:
+            repo_content_osv_data (Dict[str, Any]): A dictionary with osv.dev expected request
+              data format plus associated Pulp Content and RepoVersion.
+        """
+        vulns, content, repo_version = await self._process_reports(repo_content_osv_data)
+        await self._save_vulnerability_report(vulns, content, repo_version)
+
+    async def _process_reports(self, repo_content_osv_data):
+        """
+        Process vulnerability scanning data by querying the OSV API and extracting results.
+
+        Args:
+            repo_content_osv_data (Dict[str, Any]): A dictionary containing OSV.dev expected
+                request data format plus associated Pulp Content and RepositoryVersion.
+
+        Returns:
+            tuple:
+                - vulns (List[Dict]): List of vulnerability (vulns) returned from OSV.dev API
+                - pulp_content (Content): The Pulp Content instance that was scanned
+                - repo_version (RepositoryVersion or None): The RepositoryVersion where the
+                  Content is present
+        """
+        vulns = []
+        async with self.semaphore:
+            repo_version = repo_content_osv_data.pop("repo_version", None)
+            pulp_content = repo_content_osv_data.pop("content")
+            vulnerability_data = await self._query_osv_api(repo_content_osv_data)
+            vulns = vulnerability_data.get("vulns", [])
+            while next_page_token := vulnerability_data.get("next_page_token"):
+                repo_content_osv_data["page_token"] = next_page_token
+                vulnerability_data = await self._query_osv_api(repo_content_osv_data)
+                vulns.extend(vulnerability_data)
+        await self.total_scanned.aincrement()
+        return vulns, pulp_content, repo_version
+
+    async def _query_osv_api(self, osv_data):
+        """
+        Make a single HTTP POST request to the OSV.dev vulnerability database API.
+
+        Args:
+            osv_data (Dict[str, Any]): OSV query data dictionary containing package information
+                in the format expected by the OSV.dev API.
+
+        Returns:
+            Dict[str, Any]: JSON response from the OSV API containing vulnerability data.
+        """
+        async with self.session.post(url=OSV_QUERY_URL, json=osv_data) as response:
+            try:
+                return await response.json()
+            except aiohttp.ContentTypeError:
+                raise RuntimeError("Vuln report task failed to query osv.dev data.")
+
+    async def _save_vulnerability_report(self, vulns, content, repo_version):
+        """
+        Save vulnerability report to the database.
+
+        Args:
+            vulns (List): osv.dev vulns output
+            content (Content): A Pulp Content instance which vulnerabilities were verified
+            repo_version (RepositoryVersion): The RepositoryVersion where Content is present
+        """
+
+        vuln_report, created = await sync_to_async(VulnerabilityReport.objects.update_or_create)(
+            vulns=vulns, pulp_domain=get_domain(), content=content
+        )
+        await vuln_report.repo_versions.aadd(repo_version)
+
+        if created:
+            await self.created.aincrement()
+        else:
+            await self.updated.aincrement()
+
+
+async def check_content(func, args):
+    """
+    Spawn tasks for each async generator value to make the API requests (scan) to osv.dev.
+
+    Args:
+        func (callable | str): A Pulp plugin function that should return an async generator with
+          the osv.dev expected request data format plus associated Pulp Content and RepoVersion.
+        args (tuple): The positional arguments to pass on to the func.
+    """
+    module_name, function_name = func.rsplit(".", 1)
+    module = importlib.import_module(module_name)
+    func = getattr(module, function_name)
+
+    semaphore = asyncio.Semaphore(settings.VULN_REPORT_TASK_LIMITER)
+    vuln_report_scanner = VulnerabilityReportScanner(semaphore, func(*args))
+    await vuln_report_scanner.scan_packages()

pulpcore/app/viewsets/__init__.py

@@ -82,6 +82,7 @@ from .user import (
     UserRoleViewSet,
 )
 from .replica import UpstreamPulpViewSet
+from .vulnerability_report import VulnerabilityReportViewSet
 from .openpgp import (
     OpenPGPDistributionViewSet,
     OpenPGPKeyringViewSet,

pulpcore/app/viewsets/vulnerability_report.py

@@ -0,0 +1,20 @@
+from rest_framework.mixins import DestroyModelMixin, ListModelMixin, RetrieveModelMixin
+
+from pulpcore.app.models.vulnerability_report import VulnerabilityReport
+from pulpcore.app.serializers.vulnerability_report import VulnerabilityReportSerializer
+from pulpcore.app.viewsets.base import NamedModelViewSet
+
+
+class VulnerabilityReportViewSet(
+    NamedModelViewSet, ListModelMixin, RetrieveModelMixin, DestroyModelMixin
+):
+
+    endpoint_name = "vuln_report"
+    queryset = VulnerabilityReport.objects.prefetch_related("repo_versions").select_related(
+        "content"
+    )
+    serializer_class = VulnerabilityReportSerializer
+
+    @classmethod
+    def routable(cls):
+        return True

pulpcore/constants.py

@@ -12,6 +12,10 @@ TASK_UNBLOCKING_LOCK = 84
 TASK_METRICS_HEARTBEAT_LOCK = 74
 STORAGE_METRICS_LOCK = 72
 
+# Reasons to send along a task worker wakeup call.
+TASK_WAKEUP_UNBLOCK = "unblock"
+TASK_WAKEUP_HANDLE = "handle"
+
 #: All valid task states.
 TASK_STATES = SimpleNamespace(
     WAITING="waiting",
@@ -127,3 +131,7 @@ CHECKPOINT_TS_FORMAT = "%Y%m%dT%H%M%SZ"
 # The upper boundary represents an unsigned 32-bit integer and prevents overflow
 ORPHAN_PROTECTION_TIME_LOWER_BOUND = 0
 ORPHAN_PROTECTION_TIME_UPPER_BOUND = 4294967295  # (2^32)-1
+
+# VULNERABILITY REPORT CONSTANTS
+# OSV API URL
+OSV_QUERY_URL = "https://api.osv.dev/v1/query"

pulpcore/content/__init__.py

@@ -7,7 +7,7 @@ import os
 
 from asgiref.sync import sync_to_async
 from aiohttp import web
-
+from gunicorn.arbiter import Arbiter
 import django
 
 
@@ -16,12 +16,13 @@ django.setup()
 
 from django.conf import settings  # noqa: E402: module level not at top of file
 from django.db.utils import (  # noqa: E402: module level not at top of file
+    IntegrityError,
     InterfaceError,
     DatabaseError,
 )
 
 from pulpcore.app.apps import pulp_plugin_configs  # noqa: E402: module level not at top of file
-from pulpcore.app.models import ContentAppStatus  # noqa: E402: module level not at top of file
+from pulpcore.app.models import AppStatus  # noqa: E402: module level not at top of file
 from pulpcore.app.util import get_worker_name  # noqa: E402: module level not at top of file
 
 from .handler import Handler  # noqa: E402: module level not at top of file
@@ -48,40 +49,40 @@ CONTENT_MODULE_NAME = "content"
 
 
 async def _heartbeat():
-    content_app_status = None
     name = get_worker_name()
     heartbeat_interval = settings.CONTENT_APP_TTL // 4
     msg = "Content App '{name}' heartbeat written, sleeping for '{interarrival}' seconds".format(
         name=name, interarrival=heartbeat_interval
     )
-    fail_msg = (
-        "Content App '{name}' failed to write a heartbeat to the database, sleeping for "
-        "'{interarrival}' seconds."
-    ).format(name=name, interarrival=heartbeat_interval)
+    fail_msg = ("Content App '{name}' failed to write a heartbeat to the database.").format(
+        name=name
+    )
     versions = {app.label: app.version for app in pulp_plugin_configs()}
 
+    try:
+        app_status = await AppStatus.objects.acreate(
+            name=name, app_type="content", versions=versions
+        )
+    except IntegrityError:
+        log.error(f"A content app with name {name} already exists in the database.")
+        exit(Arbiter.WORKER_BOOT_ERROR)
     try:
         while True:
+            await asyncio.sleep(heartbeat_interval)
             try:
-                content_app_status, created = await ContentAppStatus.objects.aget_or_create(
-                    name=name, defaults={"versions": versions}
-                )
-
-                if not created:
-                    await sync_to_async(content_app_status.save_heartbeat)()
-
-                    if content_app_status.versions != versions:
-                        content_app_status.versions = versions
-                        await content_app_status.asave(update_fields=["versions"])
-
+                await app_status.asave_heartbeat()
                 log.debug(msg)
             except (InterfaceError, DatabaseError):
                 await sync_to_async(Handler._reset_db_connection)()
-                log.info(fail_msg)
-            await asyncio.sleep(heartbeat_interval)
+                try:
+                    await app_status.asave_heartbeat()
+                    log.debug(msg)
+                except (InterfaceError, DatabaseError):
+                    log.error(fail_msg)
+                    exit(Arbiter.WORKER_BOOT_ERROR)
     finally:
-        if content_app_status:
-            await content_app_status.adelete()
+        if app_status:
+            await app_status.adelete()
 
 
 async def _heartbeat_ctx(app):

pulpcore/content/handler.py

@@ -705,8 +705,11 @@ class Handler:
         rel_path = rel_path[len(distro.base_path) :]
         rel_path = rel_path.lstrip("/")
 
-        if rel_path == "" and not path.endswith("/"):
-            # The root of a distribution base_path was requested without a slash
+        # Check if we need to redirect to add a trailing slash for distro root
+        if not path.endswith("/") and (
+            rel_path == ""  # Distro root
+            or (distro.checkpoint and "/" not in rel_path)  # Timestamped checkpoint root
+        ):
             raise HTTPMovedPermanently(f"{request.path}/")
 
         original_rel_path = rel_path

pulpcore/migrations.py

@@ -3,6 +3,8 @@ from packaging.version import parse as parse_version
 from django.conf import settings
 from django.utils import timezone
 from django.db.migrations.operations.base import Operation
+from django.db.models import F
+from django.db.models.functions import Now
 
 
 class RequireVersion(Operation):
@@ -22,21 +24,41 @@ class RequireVersion(Operation):
 
     def database_forwards(self, app_label, schema_editor, from_state, to_state):
         from_state.clear_delayed_apps_cache()
-        ApiAppStatus = from_state.apps.get_model("core", "ApiAppStatus")
-        ContentAppStatus = from_state.apps.get_model("core", "ContentAppStatus")
-        Worker = from_state.apps.get_model("core", "Worker")
 
         needed_version = parse_version(self.version)
         errors = []
+        found_either_table = False
 
-        for worker_class, ttl, class_name in [
-            (ApiAppStatus, settings.API_APP_TTL, "api server"),
-            (ContentAppStatus, settings.CONTENT_APP_TTL, "content server"),
-            (Worker, settings.WORKER_TTL, "pulp worker"),
-        ]:
-            for worker in worker_class.objects.filter(
-                last_heartbeat__gte=timezone.now() - timezone.timedelta(seconds=ttl)
-            ):
+        try:
+            ApiAppStatus = from_state.apps.get_model("core", "ApiAppStatus")
+            ContentAppStatus = from_state.apps.get_model("core", "ContentAppStatus")
+            Worker = from_state.apps.get_model("core", "Worker")
+        except LookupError:
+            pass
+        else:
+            for worker_class, ttl, class_name in [
+                (ApiAppStatus, settings.API_APP_TTL, "api server"),
+                (ContentAppStatus, settings.CONTENT_APP_TTL, "content server"),
+                (Worker, settings.WORKER_TTL, "pulp worker"),
+            ]:
+                for worker in worker_class.objects.filter(
+                    last_heartbeat__gte=timezone.now() - timezone.timedelta(seconds=ttl)
+                ):
+                    present_version = worker.versions.get(self.plugin)
+                    if (
+                        present_version is not None
+                        and parse_version(present_version) < needed_version
+                    ):
+                        errors.append(
+                            f"  - '{self.plugin}'='{present_version}' "
+                            f"with {class_name} '{worker.name}'"
+                        )
+
+            found_either_table = True
+
+        try:
+            AppStatus = from_state.apps.get_model("core", "AppStatus")
+            for worker in AppStatus.objects.filter(F("last_heartbeat") + F("ttl") >= Now()):
                 present_version = worker.versions.get(self.plugin)
                 if present_version is not None and parse_version(present_version) < needed_version:
                     errors.append(
@@ -44,6 +66,11 @@ class RequireVersion(Operation):
                         f"with {class_name} '{worker.name}'"
                     )
 
+            found_either_table = True
+        except LookupError:
+            pass
+
+        assert found_either_table
         if errors:
             raise RuntimeError(
                 "\n".join(

pulpcore/openapi/__init__.py

@@ -468,6 +468,14 @@ class BasicAuthenticationScheme(OpenApiAuthenticationExtension):
         }
 
 
+class PulpRemoteUserAuthenticationScheme(OpenApiAuthenticationExtension):
+    target_class = "pulpcore.app.authentication.PulpRemoteUserAuthentication"
+    name = "remoteUserAuthentication"
+
+    def get_security_definition(self, auto_schema):
+        return settings.REMOTE_USER_OPENAPI_SECURITY_SCHEME
+
+
 class JSONHeaderRemoteAuthenticationScheme(OpenApiAuthenticationExtension):
     target_class = "pulpcore.app.authentication.JSONHeaderRemoteAuthentication"
     name = "json_header_remote_authentication"

pulpcore/plugin/models/__init__.py

@@ -40,6 +40,7 @@ from pulpcore.app.models import (
     TaskGroup,
     Upload,
     UploadChunk,
+    VulnerabilityReport,
 )
 
 
@@ -87,4 +88,5 @@ __all__ = [
     "UploadChunk",
     "EncryptedTextField",
     "system_id",
+    "VulnerabilityReport",
 ]

pulpcore/plugin/serializers/__init__.py

@@ -38,6 +38,7 @@ from pulpcore.app.serializers import (
     ValidateFieldsMixin,
     validate_unknown_fields,
     TaskSerializer,
+    VulnerabilityReportSerializer,
 )
 
 from .content import (
@@ -87,4 +88,5 @@ __all__ = [
     "TaskSerializer",
     "NoArtifactContentUploadSerializer",
     "SingleArtifactContentUploadSerializer",
+    "VulnerabilityReportSerializer",
 ]

pulpcore/plugin/tasking.py

@@ -12,11 +12,13 @@ from pulpcore.app.tasks import (
     orphan_cleanup,
     reclaim_space,
 )
+from pulpcore.app.tasks.vulnerability_report import check_content
 from pulpcore.app.tasks.repository import add_and_remove
 
 
 __all__ = [
     "ageneral_update",
+    "check_content",
     "dispatch",
     "fs_publication_export",
     "fs_repo_version_export",

pulpcore/plugin/viewsets/__init__.py

@@ -33,6 +33,7 @@ from pulpcore.app.viewsets import (
     RolesMixin,
     TaskGroupViewSet,
     TaskViewSet,
+    VulnerabilityReportViewSet,
 )
 
 from pulpcore.app.viewsets.custom_filters import (
@@ -89,4 +90,5 @@ __all__ = [
     "NoArtifactContentViewSet",
     "NoArtifactContentUploadViewSet",
     "SingleArtifactContentUploadViewSet",
+    "VulnerabilityReportViewSet",
 ]

pulpcore/pytest_plugin.py

@@ -608,28 +608,28 @@ def backend_settings_factory(pulp_settings):
             "bucket_name",
         ]
         keys["storages.backends.azure_storage.AzureStorage"] = [
-            "AZURE_ACCOUNT_NAME",
-            "AZURE_CONTAINER",
-            "AZURE_ACCOUNT_KEY",
-            "AZURE_URL_EXPIRATION_SECS",
-            "AZURE_OVERWRITE_FILES",
-            "AZURE_LOCATION",
-            "AZURE_CONNECTION_STRING",
+            "account_name",
+            "azure_container",
+            "account_key",
+            "expiration_secs",
+            "overwrite_files",
+            "location",
+            "connection_string",
         ]
-        settings = storage_settings or dict()
-        backend = storage_class or pulp_settings.STORAGES["default"]["BACKEND"]
-        not_defined_settings = (k for k in keys[backend] if k not in settings)
-        # The CI configures s3 with STORAGES and Azure with legacy
-        # Move all to STORAGES structure on DEFAULT_FILE_STORAGE removal
-        if backend == "storages.backends.s3boto3.S3Boto3Storage":
-            storages_dict = getattr(pulp_settings, "STORAGES", {})
-            storage_options = storages_dict.get("default", {}).get("OPTIONS", {})
-            for key in not_defined_settings:
-                settings[key] = storage_options.get(key)
-        else:
-            for key in not_defined_settings:
-                settings[key] = getattr(pulp_settings, key, None)
-        return backend, settings
+
+        def get_installation_storage_option(key, backend):
+            value = pulp_settings.STORAGES["default"]["OPTIONS"].get(key)
+            # Some FileSystem backend options may be defined in the top settings module
+            if backend == "pulpcore.app.models.storage.FileSystem" and not value:
+                value = getattr(pulp_settings, key, None)
+            return value
+
+        storage_settings = storage_settings or dict()
+        storage_backend = storage_class or pulp_settings.STORAGES["default"]["BACKEND"]
+        unset_storage_settings = (k for k in keys[storage_backend] if k not in storage_settings)
+        for key in unset_storage_settings:
+            storage_settings[key] = get_installation_storage_option(key, storage_backend)
+        return storage_backend, storage_settings
 
     return _settings_factory
 

pulpcore/tasking/entrypoint.py

@@ -21,8 +21,18 @@ _logger = logging.getLogger(__name__)
 @click.option(
     "--reload/--no-reload", help="Reload worker on code changes. [requires hupper to be installed.]"
 )
+@click.option(
+    "--auxiliary/--no-auxiliary",
+    default=False,
+    help="Auxiliary workers do not perform housekeeping duties.",
+)
 @click.command()
-def worker(pid, burst, reload):
+def worker(
+    pid,
+    burst,
+    reload,
+    auxiliary,
+):
     """A Pulp worker."""
 
     if reload:
@@ -40,4 +50,4 @@ def worker(pid, burst, reload):
 
     _logger.info("Starting distributed type worker")
 
-    PulpcoreWorker().run(burst=burst)
+    PulpcoreWorker(auxiliary=auxiliary).run(burst=burst)

pulpcore/tasking/tasks.py

@@ -9,12 +9,11 @@ import traceback
 import tempfile
 import threading
 from asgiref.sync import sync_to_async
-from datetime import timedelta
 from gettext import gettext as _
 
 from django.conf import settings
 from django.db import connection, transaction
-from django.db.models import Model, Max
+from django.db.models import Model
 from django_guid import get_guid
 from pulpcore.app.apps import MODULE_PLUGIN_VERSIONS
 from pulpcore.app.models import Task, TaskGroup
@@ -23,8 +22,8 @@ from pulpcore.constants import (
     TASK_FINAL_STATES,
     TASK_INCOMPLETE_STATES,
     TASK_STATES,
-    TASK_DISPATCH_LOCK,
     IMMEDIATE_TIMEOUT,
+    TASK_WAKEUP_UNBLOCK,
 )
 from pulpcore.middleware import x_task_diagnostics_var
 from pulpcore.tasking.kafka import send_task_notification
@@ -47,10 +46,10 @@ def _validate_and_get_resources(resources):
     return list(resource_set)
 
 
-def wakeup_worker():
+def wakeup_worker(reason="unknown"):
     # Notify workers
     with connection.connection.cursor() as cursor:
-        cursor.execute("NOTIFY pulp_worker_wakeup")
+        cursor.execute("SELECT pg_notify('pulp_worker_wakeup', %s)", (reason,))
 
 
 def execute_task(task):
@@ -228,13 +227,6 @@ def dispatch(
     notify_workers = False
     with contextlib.ExitStack() as stack:
         with transaction.atomic():
-            # Task creation need to be serialized so that pulp_created will provide a stable order
-            # at every time. We specifically need to ensure that each task, when committed to the
-            # task table will be the newest with respect to `pulp_created`.
-            with connection.cursor() as cursor:
-                # Wait for exclusive access and release automatically after transaction.
-                cursor.execute("SELECT pg_advisory_xact_lock(%s, %s)", [0, TASK_DISPATCH_LOCK])
-            newest_created = Task.objects.aggregate(Max("pulp_created"))["pulp_created__max"]
             task = Task.objects.create(
                 state=TASK_STATES.WAITING,
                 logging_cid=(get_guid()),
@@ -250,23 +242,6 @@ def dispatch(
                 profile_options=x_task_diagnostics_var.get(None),
             )
             task.refresh_from_db()  # The database may have assigned a timestamp for us.
-            if newest_created and task.pulp_created <= newest_created:
-                # Let this workaround not row forever into the future.
-                if newest_created - task.pulp_created > timedelta(seconds=1):
-                    # Do not commit the transaction if this condition is not met.
-                    # If we ever hit this, think about delegating the timestamping to PostgresQL.
-                    raise RuntimeError("Clockscrew detected. Task dispatching would be dangerous.")
-                # Try to work around the smaller glitch
-                task.pulp_created = newest_created + timedelta(milliseconds=1)
-                task.save()
-            if task_group:
-                task_group.refresh_from_db()
-                if task_group.all_tasks_dispatched:
-                    task.set_canceling()
-                    task.set_canceled(
-                        TASK_STATES.CANCELED, "All tasks in group have been dispatched/canceled."
-                    )
-                    return task
             if immediate:
                 # Grab the advisory lock before the task hits the db.
                 stack.enter_context(task)
@@ -308,7 +283,7 @@ def dispatch(
                 task.set_canceling()
                 task.set_canceled(TASK_STATES.CANCELED, "Resources temporarily unavailable.")
     if notify_workers:
-        wakeup_worker()
+        wakeup_worker(TASK_WAKEUP_UNBLOCK)
     return task