pulpcore 3.89.1__py3-none-any.whl → 3.90.0__py3-none-any.whl

pulpcore/app/models/__init__.py

@@ -69,7 +69,7 @@ from .repository import (
     RepositoryVersionContentDetails,
 )
 
-from .status import AppStatus, ApiAppStatus, ContentAppStatus
+from .status import AppStatus
 
 from .task import (
     CreatedResource,
@@ -77,7 +77,6 @@ from .task import (
     Task,
     TaskGroup,
     TaskSchedule,
-    Worker,
 )
 
 from .analytics import SystemID
@@ -106,6 +105,7 @@ from .openpgp import (
 )
 
 __all__ = [
+    "AppStatus",
     "BaseModel",
     "MasterModel",
     "pulp_uuid",
@@ -151,14 +151,11 @@ __all__ = [
     "RepositoryContent",
     "RepositoryVersion",
     "RepositoryVersionContentDetails",
-    "ApiAppStatus",
-    "ContentAppStatus",
     "CreatedResource",
     "ProfileArtifact",
     "Task",
     "TaskGroup",
     "TaskSchedule",
-    "Worker",
     "SystemID",
     "Upload",
     "UploadChunk",

pulpcore/app/models/exporter.py

@@ -9,6 +9,7 @@ from pulpcore.app.models import (
     MasterModel,
 )
 from pulpcore.app.models.repository import Repository
+from pulpcore.app.util import get_domain_pk
 from pulpcore.constants import FS_EXPORT_CHOICES, FS_EXPORT_METHODS
 
 
@@ -29,6 +30,7 @@ class Export(BaseModel):
     params = models.JSONField(null=True)
     task = models.ForeignKey("Task", on_delete=models.SET_NULL, null=True)
     exporter = models.ForeignKey("Exporter", on_delete=models.CASCADE)
+    pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT)
 
 
 class ExportedResource(GenericRelationModel):
@@ -54,7 +56,11 @@ class Exporter(MasterModel):
         name (models.TextField): The exporter unique name.
     """
 
-    name = models.TextField(db_index=True, unique=True)
+    name = models.TextField()
+    pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT)
+
+    class Meta:
+        unique_together = ("name", "pulp_domain")
 
 
 class FilesystemExport(Export):

pulpcore/app/models/fields.py

@@ -79,7 +79,6 @@ class ArtifactFileField(FileField):
                     "prior to Artifact creation."
                 )
             )
-
         move = file._committed and file.name != artifact_storage_path
         if move:
             if not already_in_place:

pulpcore/app/models/importer.py

@@ -5,6 +5,8 @@ from pulpcore.app.models import (
     BaseModel,
     MasterModel,
 )
+from pulpcore.app.util import get_domain_pk
+
 from .repository import Repository
 
 
@@ -23,6 +25,7 @@ class Import(BaseModel):
     params = models.JSONField(null=True)
     task = models.ForeignKey("Task", on_delete=models.PROTECT)
     importer = models.ForeignKey("Importer", on_delete=models.CASCADE)
+    pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT)
 
 
 class Importer(MasterModel):
@@ -35,7 +38,11 @@ class Importer(MasterModel):
         name (models.TextField): The importer unique name.
     """
 
-    name = models.TextField(db_index=True, unique=True)
+    name = models.TextField()
+    pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT)
+
+    class Meta:
+        unique_together = ("name", "pulp_domain")
 
 
 class PulpImporter(Importer):

pulpcore/app/models/repository.py

@@ -381,6 +381,21 @@ class Repository(MasterModel):
         body = {"repository_pk": self.pk, "add_content_units": [cpk], "remove_content_units": []}
         return dispatch(add_and_remove, kwargs=body, exclusive_resources=[self], immediate=True)
 
+    async def async_pull_through_add_content(self, content_artifact):
+        cpk = content_artifact.content_id
+        already_present = RepositoryContent.objects.filter(
+            content__pk=cpk, repository=self, version_removed__isnull=True
+        )
+        if not cpk or await already_present.aexists():
+            return None
+
+        from pulpcore.plugin.tasking import adispatch, aadd_and_remove
+
+        body = {"repository_pk": self.pk, "add_content_units": [cpk], "remove_content_units": []}
+        return await adispatch(
+            aadd_and_remove, kwargs=body, exclusive_resources=[self], immediate=True
+        )
+
     @hook(AFTER_UPDATE, when="retain_repo_versions", has_changed=True)
     def _cleanup_old_versions_hook(self):
         # Do not attempt to clean up anything, while there is a transaction involving repo versions
@@ -1105,6 +1120,7 @@ class RepositoryVersion(BaseModel):
 
         if not content or not content.count():
             return
+
         assert (
             not Content.objects.filter(pk__in=content)
             .exclude(pulp_domain_id=get_domain_pk())

pulpcore/app/models/status.py

@@ -15,44 +15,12 @@ from pulpcore.app.models import BaseModel
 
 
 class AppStatusManager(models.Manager):
-    # This should be replaced with 3.87.
-    def online(self):
-        """
-        Returns a queryset of objects that are online.
-
-        To be considered 'online', a AppStatus must have a heartbeat timestamp within
-        ``self.model.APP_TTL`` from now.
-
-        Returns:
-            [django.db.models.query.QuerySet][]:  A query set of the
-                objects which are considered 'online'.
-        """
-        age_threshold = timezone.now() - self.model.APP_TTL
-        return self.filter(last_heartbeat__gte=age_threshold)
-
-    def missing(self, age=None):
-        """
-        Returns a queryset of workers meeting the criteria to be considered 'missing'
-
-        To be considered missing, a AppsStatus must have a stale timestamp.  By default, stale is
-        defined here as longer than the ``self.model.APP_TTL``, or you can specify age as a
-        timedelta.
-
-        Args:
-            age (datetime.timedelta): Objects who have heartbeats older than this time interval are
-                considered missing.
-
-        Returns:
-            [django.db.models.query.QuerySet][]:  A query set of the objects objects which
-                are considered to be 'missing'.
-        """
-        age_threshold = timezone.now() - (age or self.model.APP_TTL)
-        return self.filter(last_heartbeat__lt=age_threshold)
-
+    _APP_TTL = {
+        "api": settings.API_APP_TTL,
+        "content": settings.CONTENT_APP_TTL,
+        "worker": settings.WORKER_TTL,
+    }
 
-class _AppStatusManager(AppStatusManager):
-    # This is an intermediate class in order to allow a ZDU.
-    # It should be made the real thing with 3.87.
     def __init__(self):
         super().__init__()
         self._current_app_status = None
@@ -61,6 +29,7 @@ class _AppStatusManager(AppStatusManager):
         if self._current_app_status is not None:
             raise RuntimeError("There is already an app status in this process.")
 
+        kwargs.setdefault("ttl", timedelta(seconds=self._APP_TTL[app_type]))
         obj = super().create(app_type=app_type, **kwargs)
         self._current_app_status = obj
         return obj
@@ -69,6 +38,7 @@ class _AppStatusManager(AppStatusManager):
         if self._current_app_status is not None:
             raise RuntimeError("There is already an app status in this process.")
 
+        kwargs.setdefault("ttl", timedelta(seconds=self._APP_TTL[app_type]))
         obj = await super().acreate(app_type=app_type, **kwargs)
         self._current_app_status = obj
         return obj
@@ -103,12 +73,7 @@ class AppStatus(BaseModel):
         ("content", "content"),
         ("worker", "worker"),
     ]
-    _APP_TTL = {
-        "api": settings.API_APP_TTL,
-        "content": settings.CONTENT_APP_TTL,
-        "worker": settings.WORKER_TTL,
-    }
-    objects = _AppStatusManager()
+    objects = AppStatusManager()
 
     app_type = models.CharField(max_length=10, choices=APP_TYPES)
     name = models.TextField()
@@ -116,10 +81,6 @@ class AppStatus(BaseModel):
     ttl = models.DurationField(null=False)
     last_heartbeat = models.DateTimeField(auto_now=True)
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.ttl = timedelta(seconds=self._APP_TTL[self.app_type])
-
     @property
     def online(self) -> bool:
         """
@@ -170,94 +131,3 @@ class AppStatus(BaseModel):
         The task this worker is currently executing, if any.
         """
         return self.tasks.filter(state="running").first()
-
-
-class BaseAppStatus(BaseModel):
-    """
-    Represents an AppStatus.
-    Deprecated, to be removed with 3.87.
-
-    This class is abstract. Subclasses must define `APP_TTL` as a `timedelta`.
-
-    Fields:
-
-        name (models.TextField): The name of the app.
-        last_heartbeat (models.DateTimeField): A timestamp of this worker's last heartbeat.
-        versions (HStoreField): A dictionary with versions of all pulp components.
-    """
-
-    objects = AppStatusManager()
-
-    name = models.TextField(db_index=True, unique=True)
-    last_heartbeat = models.DateTimeField(auto_now=True)
-    versions = HStoreField(default=dict)
-
-    @property
-    def online(self):
-        """
-        Whether an app can be considered 'online'
-
-        To be considered 'online', an app must have a timestamp more recent than ``self.APP_TTL``.
-
-        Returns:
-            bool: True if the app is considered online, otherwise False
-        """
-        age_threshold = timezone.now() - self.APP_TTL
-        return self.last_heartbeat >= age_threshold
-
-    @property
-    def missing(self):
-        """
-        Whether an app can be considered 'missing'
-
-        To be considered 'missing', an App must have a timestamp older than ``self.APP_TTL``.
-
-        Returns:
-            bool: True if the app is considered missing, otherwise False
-        """
-        return not self.online
-
-    def save_heartbeat(self):
-        """
-        Update the last_heartbeat field to now and save it.
-
-        Only the last_heartbeat field will be saved. No other changes will be saved.
-
-        Raises:
-            ValueError: When the model instance has never been saved before. This method can
-                only update an existing database record.
-        """
-        self.save(update_fields=["last_heartbeat"])
-
-    async def asave_heartbeat(self):
-        """
-        Update the last_heartbeat field to now and save it.
-
-        Only the last_heartbeat field will be saved. No other changes will be saved.
-
-        Raises:
-            ValueError: When the model instance has never been saved before. This method can
-                only update an existing database record.
-        """
-        await self.asave(update_fields=["last_heartbeat"])
-
-    class Meta:
-        abstract = True
-
-
-class ApiAppStatus(BaseAppStatus):
-    """
-    Represents a Api App Status
-    Deprecated, to be removed with 3.87.
-    """
-
-    APP_TTL = timedelta(seconds=settings.API_APP_TTL)
-
-
-class ContentAppStatus(BaseAppStatus):
-    """
-    Represents a Content App Status
-    Deprecated, to be removed with 3.87.
-    """
-
-    APP_TTL = timedelta(seconds=settings.CONTENT_APP_TTL)

pulpcore/app/models/task.py

@@ -5,10 +5,8 @@ Django models related to the Tasking system
 import json
 import logging
 import traceback
-from datetime import timedelta
 from gettext import gettext as _
 
-from django.conf import settings
 from django.contrib.postgres.fields import ArrayField, HStoreField
 from django.contrib.postgres.indexes import GinIndex
 from django.core.serializers.json import DjangoJSONEncoder
@@ -21,7 +19,7 @@ from pulpcore.app.models import (
     BaseModel,
     GenericRelationModel,
 )
-from pulpcore.app.models.status import AppStatus, BaseAppStatus
+from pulpcore.app.models.status import AppStatus
 from pulpcore.app.models.fields import EncryptedJSONField
 from pulpcore.constants import TASK_CHOICES, TASK_INCOMPLETE_STATES, TASK_STATES
 from pulpcore.exceptions import exception_to_dict
@@ -31,25 +29,6 @@ from pulpcore.app.loggers import deprecation_logger
 _logger = logging.getLogger(__name__)
 
 
-class Worker(BaseAppStatus):
-    """
-    Represents a worker
-    Deprecated, to be removed with 3.87.
-    """
-
-    APP_TTL = timedelta(seconds=settings.WORKER_TTL)
-
-    @property
-    def current_task(self):
-        """
-        The task this worker is currently executing, if any.
-
-        Returns:
-            Task: The currently executing task
-        """
-        return self.tasks.filter(state="running").first()
-
-
 def _uuid_to_advisory_lock(value):
     return ((value >> 64) ^ value) & 0x7FFFFFFFFFFFFFFF
 
@@ -140,7 +119,6 @@ class Task(BaseModel, AutoAddObjPermsMixin):
     enc_args = EncryptedJSONField(null=True, encoder=DjangoJSONEncoder)
     enc_kwargs = EncryptedJSONField(null=True, encoder=DjangoJSONEncoder)
 
-    worker = models.ForeignKey("Worker", null=True, related_name="tasks", on_delete=models.SET_NULL)
     # This field is the lock to protect tasks.
     app_lock = models.ForeignKey(
         "AppStatus", null=True, related_name="tasks", on_delete=models.SET_NULL
@@ -363,8 +341,20 @@ class Task(BaseModel, AutoAddObjPermsMixin):
             self.unblocked_at = unblocked_at
         else:
             self.refresh_from_db()
-            raise RuntimeError(
-                _("Falied to set task {} unblocked in state '{}'.").format(self.pk, self.state)
+            raise RuntimeError("Failed to set task {} unblocked in state '{}'.").format(
+                self.pk, self.state
+            )
+
+    async def aunblock(self):
+        # This should be safe to be called without holding the lock.
+        unblocked_at = timezone.now()
+        rows = await Task.objects.filter(pk=self.pk).aupdate(unblocked_at=unblocked_at)
+        if rows == 1:
+            self.unblocked_at = unblocked_at
+        else:
+            await self.arefresh_from_db()
+            raise RuntimeError("Failed to set task {} unblocked in state '{}'.").format(
+                self.pk, self.state
             )
 
     class Meta:

pulpcore/app/serializers/domain.py

@@ -2,7 +2,7 @@ from gettext import gettext as _
 import json
 
 from django.conf import settings
-from django.core.files.storage import import_string
+from django.utils.module_loading import import_string
 from django.core.exceptions import ImproperlyConfigured
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import extend_schema_field

pulpcore/app/serializers/exporter.py

@@ -3,12 +3,12 @@ from gettext import gettext as _
 import re
 
 from rest_framework import serializers
-from rest_framework.validators import UniqueValidator
 
 from pulpcore.app import models, settings
 from pulpcore.app.serializers import (
     DetailIdentityField,
     DetailRelatedField,
+    DomainUniqueValidator,
     ExportIdentityField,
     ExportRelatedField,
     ModelSerializer,
@@ -37,8 +37,8 @@ class ExporterSerializer(ModelSerializer):
 
     pulp_href = DetailIdentityField(view_name_pattern=r"exporter(-.*/.*)-detail")
     name = serializers.CharField(
-        help_text=_("Unique name of the file system exporter."),
-        validators=[UniqueValidator(queryset=models.Exporter.objects.all())],
+        help_text=_("Unique name of the exporter."),
+        validators=[DomainUniqueValidator(queryset=models.Exporter.objects.all())],
     )
 
     @staticmethod
@@ -308,7 +308,7 @@ class FilesystemExportSerializer(ExportSerializer):
             raise serializers.ValidationError(
                 _("publication or repository_version must either be supplied but not both.")
             )
-        return data
+        return super().validate(data)
 
     class Meta:
         model = models.FilesystemExport

pulpcore/app/serializers/importer.py

@@ -3,11 +3,11 @@ from gettext import gettext as _
 
 from django.core.exceptions import ObjectDoesNotExist
 from rest_framework import serializers
-from rest_framework.validators import UniqueValidator
 
 from pulpcore.app import models, settings
 from pulpcore.app.serializers import (
     DetailIdentityField,
+    DomainUniqueValidator,
     ImportIdentityField,
     ModelSerializer,
     RelatedField,
@@ -22,7 +22,7 @@ class ImporterSerializer(ModelSerializer):
     pulp_href = DetailIdentityField(view_name_pattern=r"importer(-.*/.*)-detail")
     name = serializers.CharField(
         help_text=_("Unique name of the Importer."),
-        validators=[UniqueValidator(queryset=models.Importer.objects.all())],
+        validators=[DomainUniqueValidator(queryset=models.Importer.objects.all())],
     )
 
     class Meta:

pulpcore/app/serializers/task.py

@@ -1,6 +1,9 @@
+import typing as t
 from gettext import gettext as _
 
 from rest_framework import serializers
+from drf_spectacular.utils import extend_schema_serializer
+from drf_spectacular.types import OpenApiTypes
 
 from pulpcore.app import models
 from pulpcore.app.serializers import (
@@ -52,13 +55,9 @@ class TaskSerializer(ModelSerializer):
         ),
         read_only=True,
     )
-    worker = RelatedField(
-        help_text=_(
-            "The worker associated with this task."
-            " This field is empty if a worker is not yet assigned."
-        ),
+    worker = serializers.SerializerMethodField(
+        help_text=_("DEPRECATED - Always null"),
         read_only=True,
-        view_name="workers-detail",
     )
     parent_task = RelatedField(
         help_text=_("The parent task that spawned this task."),
@@ -93,7 +92,10 @@ class TaskSerializer(ModelSerializer):
         help_text=_("The result of this task."),
     )
 
-    def get_created_by(self, obj):
+    def get_worker(self, obj) -> t.Optional[OpenApiTypes.URI]:
+        return None
+
+    def get_created_by(self, obj) -> t.Optional[OpenApiTypes.URI]:
         if task_user_map := self.context.get("task_user_mapping"):
             if user_id := task_user_map.get(str(obj.pk)):
                 kwargs = {"pk": user_id}
@@ -136,11 +138,12 @@ class MinimalTaskSerializer(TaskSerializer):
         )
 
 
+@extend_schema_serializer(deprecate_fields=["all_tasks_dispatched"])
 class TaskGroupSerializer(ModelSerializer):
     pulp_href = IdentityField(view_name="task-groups-detail")
     description = serializers.CharField(help_text=_("A description of the task group."))
     all_tasks_dispatched = serializers.BooleanField(
-        help_text=_("Whether all tasks have been spawned for this task group.")
+        help_text=_("Whether all tasks have been spawned for this task group."),
     )
 
     waiting = TaskGroupStatusCountField(

pulpcore/app/tasks/importer.py

@@ -35,7 +35,12 @@ from pulpcore.app.modelresource import (
     ContentArtifactResource,
     RepositoryResource,
 )
-from pulpcore.app.util import compute_file_hash, Crc32Hasher
+from pulpcore.app.util import (
+    compute_file_hash,
+    Crc32Hasher,
+    get_domain,
+    get_domain_pk,
+)
 from pulpcore.constants import TASK_STATES
 from pulpcore.tasking.tasks import dispatch
 
@@ -435,7 +440,6 @@ def pulp_import(importer_pk, path, toc, create_repositories):
         create_repositories (bool): Indicates whether missing repositories should be automatically
             created or not.
     """
-
     if toc:
         path = toc
         fileobj = ChunkedFile(toc)
@@ -443,13 +447,12 @@ def pulp_import(importer_pk, path, toc, create_repositories):
         fileobj.validate_chunks()
     else:
         fileobj = nullcontext()
-
     log.info(_("Importing {}.").format(path))
     current_task = Task.current()
     task_group = TaskGroup.current()
     importer = PulpImporter.objects.get(pk=importer_pk)
     the_import = PulpImport.objects.create(
-        importer=importer, task=current_task, params={"path": path}
+        importer=importer, task=current_task, params={"path": path}, pulp_domain=get_domain()
     )
     CreatedResource.objects.create(content_object=the_import)
 
@@ -485,18 +488,47 @@ def pulp_import(importer_pk, path, toc, create_repositories):
             message="Importing Artifacts",
             code="import.artifacts",
         )
+        into_default = "default" == get_domain().name
         with ProgressReport(**data) as pb:
             # Import artifacts, and place their binary blobs, one batch at a time.
             # Skip artifacts that already exist in storage.
             for ar_result in _import_file(os.path.join(temp_dir, ARTIFACT_FILE), ArtifactResource):
                 for row in pb.iter(ar_result.rows):
                     artifact = Artifact.objects.get(pk=row.object_id)
-                    base_path = os.path.join("artifact", artifact.sha256[0:2], artifact.sha256[2:])
-                    src = os.path.join(temp_dir, base_path)
 
-                    if not default_storage.exists(base_path):
-                        with open(src, "rb") as f:
-                            default_storage.save(base_path, f)
+                    # If we are domain-enabled, and not importing into "default", then the
+                    # destination is "to the current domain's artifact directory". Otherwise, it's
+                    # just to /artifact/.
+                    if settings.DOMAIN_ENABLED and not into_default:
+                        destination_path = os.path.join(
+                            "artifact",
+                            str(get_domain_pk()),
+                            artifact.sha256[0:2],
+                            artifact.sha256[2:],
+                        )
+
+                    else:
+                        destination_path = os.path.join(
+                            "artifact", artifact.sha256[0:2], artifact.sha256[2:]
+                        )
+
+                    # If *the upstream* was domain-enabled, the tarfile will have artifact/DOMAIN/
+                    # in its path, and the Artifact will have artifact/current-domain-id in its
+                    # "file" attribute. We need to copy from artifact/DOMAIN/ in the tarfile.
+                    domain_path = os.path.join(
+                        "artifact", "DOMAIN", artifact.sha256[0:2], artifact.sha256[2:]
+                    )
+                    if os.path.exists(os.path.join(temp_dir, domain_path)):
+                        tar_path = domain_path
+                    else:
+                        tar_path = os.path.join(
+                            "artifact", artifact.sha256[0:2], artifact.sha256[2:]
+                        )
+                    src_in_tar = os.path.join(temp_dir, tar_path)
+
+                    if not default_storage.exists(destination_path):
+                        with open(src_in_tar, "rb") as f:
+                            default_storage.save(destination_path, f)
 
         # Now import repositories, in parallel.
 
@@ -529,7 +561,9 @@ def pulp_import(importer_pk, path, toc, create_repositories):
                 worker_rsrc = f"import-worker-{index % import_workers}"
                 exclusive_resources = [worker_rsrc]
                 try:
-                    dest_repo = Repository.objects.get(name=dest_repo_name)
+                    dest_repo = Repository.objects.get(
+                        name=dest_repo_name, pulp_domain=get_domain()
+                    )
                 except Repository.DoesNotExist:
                     if create_repositories:
                         dest_repo_pk = ""

pulpcore/app/tasks/repository.py

@@ -236,3 +236,30 @@ def add_and_remove(repository_pk, add_content_units, remove_content_units, base_
     with repository.new_version(base_version=base_version) as new_version:
         new_version.remove_content(models.Content.objects.filter(pk__in=remove_content_units))
         new_version.add_content(models.Content.objects.filter(pk__in=add_content_units))
+
+
+async def aadd_and_remove(
+    repository_pk, add_content_units, remove_content_units, base_version_pk=None
+):
+    """Aynsc version of add_and_remove."""
+    repository = await models.Repository.objects.aget(pk=repository_pk)
+    repository = await repository.acast()
+
+    if base_version_pk:
+        base_version = await models.RepositoryVersion.objects.aget(pk=base_version_pk)
+    else:
+        base_version = None
+
+    if "*" in remove_content_units:
+        latest = await repository.alatest_version()
+        if latest:
+            remove_content_units = latest.content.values_list("pk", flat=True)
+        else:
+            remove_content_units = []
+
+    def add_to_repository():
+        with repository.new_version(base_version=base_version) as new_version:
+            new_version.remove_content(models.Content.objects.filter(pk__in=remove_content_units))
+            new_version.add_content(models.Content.objects.filter(pk__in=add_content_units))
+
+    await sync_to_async(add_to_repository)()