pulp-python 3.15.1__tar.gz → 3.17.0__tar.gz

{pulp_python-3.15.1 → pulp_python-3.17.0}/CHANGES.md

@@ -8,6 +8,29 @@
 
 [//]: # (towncrier release notes start)
 
+## 3.17.0 (2025-07-23) {: #3.17.0 }
+
+#### Features {: #3.17.0-feature }
+
+- Added full support for the latest core metadata (up to 2.4).
+  [#689](https://github.com/pulp/pulp_python/issues/689)
+
+---
+
+## 3.16.0 (2025-06-10) {: #3.16.0 }
+
+#### Features {: #3.16.0-feature }
+
+- Added support for on-demand content to `repair_metadata` endpoint.
+  [#849](https://github.com/pulp/pulp_python/issues/849)
+
+#### Bugfixes {: #3.16.0-bugfix }
+
+- Fixed pull-through caching not working for indexes that use relative URLs.
+  [#842](https://github.com/pulp/pulp_python/issues/842)
+
+---
+
 ## 3.15.1 (2025-06-10) {: #3.15.1 }
 
 #### Bugfixes {: #3.15.1-bugfix }

{pulp_python-3.15.1 → pulp_python-3.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulp-python
-Version: 3.15.1
+Version: 3.17.0
 Summary: pulp-python plugin for the Pulp Project
 Author-email: Pulp Team <pulp-list@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -22,14 +22,15 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pulpcore<3.85,>=3.49.0
-Requires-Dist: pkginfo<1.13.0,>=1.10.0
+Requires-Dist: pkginfo<1.13.0,>=1.12.0
 Requires-Dist: bandersnatch<6.4,>=6.3.0
 Requires-Dist: pypi-simple<2.0,>=1.5.0
 Dynamic: license-file
 
 # pulp_python
 
-![Pulp Nightly CI/CD](https://github.com/pulp/pulp_python/actions/workflows/nightly.yml/badge.svg)
+[![PyPI](https://img.shields.io/pypi/v/pulp_python.svg)](https://pypi.python.org/pypi/pulp_python)
+[![Pulp Nightly CI/CD](https://github.com/pulp/pulp_python/actions/workflows/nightly.yml/badge.svg)](https://github.com/pulp/pulp_python/actions/workflows/nightly.yml)
 
 A Pulp plugin to support hosting your own pip compatible Python packages.
 

pulp_python-3.17.0/README.md

@@ -0,0 +1,9 @@
+# pulp_python
+
+[![PyPI](https://img.shields.io/pypi/v/pulp_python.svg)](https://pypi.python.org/pypi/pulp_python)
+[![Pulp Nightly CI/CD](https://github.com/pulp/pulp_python/actions/workflows/nightly.yml/badge.svg)](https://github.com/pulp/pulp_python/actions/workflows/nightly.yml)
+
+A Pulp plugin to support hosting your own pip compatible Python packages.
+
+For more information, please see the [documentation](https://docs.pulpproject.org/pulp_python/) or the
+[Pulp project page](https://pulpproject.org).

{pulp_python-3.15.1 → pulp_python-3.17.0}/pulp_python/app/__init__.py

@@ -10,7 +10,7 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_python.app"
     label = "python"
-    version = "3.15.1"
+    version = "3.17.0"
     python_package_name = "pulp-python"
     domain_compatible = True
 

pulp_python-3.17.0/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.2.19 on 2025-07-09 08:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('python', '0013_add_rbac_permissions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pythonpackagecontent',
+            name='dynamic',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AddField(
+            model_name='pythonpackagecontent',
+            name='license_expression',
+            field=models.TextField(default=''),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='pythonpackagecontent',
+            name='license_file',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AddField(
+            model_name='pythonpackagecontent',
+            name='provides_extras',
+            field=models.JSONField(default=list),
+        ),
+    ]

{pulp_python-3.15.1 → pulp_python-3.17.0}/pulp_python/app/models.py

@@ -140,49 +140,66 @@ class PythonPackageContent(Content):
     """
     A Content Type representing Python's Distribution Package.
 
-    As defined in pep-0426 and pep-0345.
+    Core Metadata:
+        https://packaging.python.org/en/latest/specifications/core-metadata/
 
-    https://www.python.org/dev/peps/pep-0491/
-    https://www.python.org/dev/peps/pep-0345/
-    """
-
-    PROTECTED_FROM_RECLAIM = False
+    Release metadata (JSON API):
+        https://docs.pypi.org/api/json/
 
-    TYPE = "python"
-    repo_key_fields = ("filename",)
-    # Required metadata
-    filename = models.TextField(db_index=True)
-    packagetype = models.TextField(choices=PACKAGE_TYPES)
-    name = models.TextField()
-    name.register_lookup(NormalizeName)
-    version = models.TextField()
-    sha256 = models.CharField(db_index=True, max_length=64)
-    # Optional metadata
-    python_version = models.TextField()
-    metadata_version = models.TextField()
-    summary = models.TextField()
-    description = models.TextField()
-    keywords = models.TextField()
-    home_page = models.TextField()
-    download_url = models.TextField()
+    File Formats:
+        https://packaging.python.org/en/latest/specifications/source-distribution-format/
+        https://packaging.python.org/en/latest/specifications/binary-distribution-format/
+    """
+    # Core metadata
+    # Version 1.0
     author = models.TextField()
     author_email = models.TextField()
+    description = models.TextField()
+    home_page = models.TextField()  # Deprecated in favour of Project-URL
+    keywords = models.TextField()
+    license = models.TextField()  # Deprecated in favour of License-Expression
+    metadata_version = models.TextField()
+    name = models.TextField()
+    platform = models.TextField()
+    summary = models.TextField()
+    version = models.TextField()
+    # Version 1.1
+    classifiers = models.JSONField(default=list)
+    download_url = models.TextField()  # Deprecated in favour of Project-URL
+    supported_platform = models.TextField()
+    # Version 1.2
     maintainer = models.TextField()
     maintainer_email = models.TextField()
-    license = models.TextField()
-    requires_python = models.TextField()
+    obsoletes_dist = models.JSONField(default=list)
     project_url = models.TextField()
-    platform = models.TextField()
-    supported_platform = models.TextField()
-    requires_dist = models.JSONField(default=list)
+    project_urls = models.JSONField(default=dict)
     provides_dist = models.JSONField(default=list)
-    obsoletes_dist = models.JSONField(default=list)
     requires_external = models.JSONField(default=list)
-    classifiers = models.JSONField(default=list)
-    project_urls = models.JSONField(default=dict)
+    requires_dist = models.JSONField(default=list)
+    requires_python = models.TextField()
+    # Version 2.1
     description_content_type = models.TextField()
-    # Pulp Domains
-    _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT)
+    provides_extras = models.JSONField(default=list)
+    # Version 2.2
+    dynamic = models.JSONField(default=list)
+    # Version 2.4
+    license_expression = models.TextField()
+    license_file = models.JSONField(default=list)
+
+    # Release metadata
+    filename = models.TextField(db_index=True)
+    packagetype = models.TextField(choices=PACKAGE_TYPES)
+    python_version = models.TextField()
+    sha256 = models.CharField(db_index=True, max_length=64)
+
+    # From pulpcore
+    PROTECTED_FROM_RECLAIM = False
+    TYPE = "python"
+    _pulp_domain = models.ForeignKey(
+        "core.Domain", default=get_domain_pk, on_delete=models.PROTECT
+    )
+    name.register_lookup(NormalizeName)
+    repo_key_fields = ("filename",)
 
     @staticmethod
     def init_from_artifact_and_relative_path(artifact, relative_path):

{pulp_python-3.15.1 → pulp_python-3.17.0}/pulp_python/app/serializers.py

@@ -72,69 +72,69 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
     """
     A Serializer for PythonPackageContent.
     """
-
-    filename = serializers.CharField(
-        help_text=_('The name of the distribution package, usually of the format:'
-                    ' {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}'
-                    '-{platform tag}.{packagetype}'),
-        read_only=True,
-    )
-    packagetype = serializers.CharField(
-        help_text=_('The type of the distribution package '
-                    '(e.g. sdist, bdist_wheel, bdist_egg, etc)'),
-        read_only=True,
-    )
-    name = serializers.CharField(
-        help_text=_('The name of the python project.'),
-        read_only=True,
-    )
-    version = serializers.CharField(
-        help_text=_('The packages version number.'),
-        read_only=True,
-    )
-    sha256 = serializers.CharField(
-        default='',
-        help_text=_('The SHA256 digest of this package.'),
-    )
-    metadata_version = serializers.CharField(
-        help_text=_('Version of the file format'),
-        read_only=True,
+    # Core metadata
+    # Version 1.0
+    author = serializers.CharField(
+        required=False, allow_blank=True,
+        help_text=_('Text containing the author\'s name. Contact information can also be added,'
+                    ' separated with newlines.')
     )
-    summary = serializers.CharField(
+    author_email = serializers.CharField(
         required=False, allow_blank=True,
-        help_text=_('A one-line summary of what the package does.')
+        help_text=_('The author\'s e-mail address. ')
     )
     description = serializers.CharField(
         required=False, allow_blank=True,
         help_text=_('A longer description of the package that can run to several paragraphs.')
     )
-    description_content_type = serializers.CharField(
+    home_page = serializers.CharField(
         required=False, allow_blank=True,
-        help_text=_('A string stating the markup syntax (if any) used in the distribution’s'
-                    ' description, so that tools can intelligently render the description.')
+        help_text=_('The URL for the package\'s home page.')
     )
     keywords = serializers.CharField(
         required=False, allow_blank=True,
         help_text=_('Additional keywords to be used to assist searching for the '
                     'package in a larger catalog.')
     )
-    home_page = serializers.CharField(
+    license = serializers.CharField(
         required=False, allow_blank=True,
-        help_text=_('The URL for the package\'s home page.')
+        help_text=_('Text indicating the license covering the distribution')
     )
-    download_url = serializers.CharField(
+    metadata_version = serializers.CharField(
+        help_text=_('Version of the file format'),
+        read_only=True,
+    )
+    name = serializers.CharField(
+        help_text=_('The name of the python project.'),
+        read_only=True,
+    )
+    platform = serializers.CharField(
         required=False, allow_blank=True,
-        help_text=_('Legacy field denoting the URL from which this package can be downloaded.')
+        help_text=_('A comma-separated list of platform specifications, '
+                    'summarizing the operating systems supported by the package.')
     )
-    author = serializers.CharField(
+    summary = serializers.CharField(
         required=False, allow_blank=True,
-        help_text=_('Text containing the author\'s name. Contact information can also be added,'
-                    ' separated with newlines.')
+        help_text=_('A one-line summary of what the package does.')
     )
-    author_email = serializers.CharField(
+    version = serializers.CharField(
+        help_text=_('The packages version number.'),
+        read_only=True,
+    )
+    # Version 1.1
+    classifiers = serializers.JSONField(
+        required=False, default=list,
+        help_text=_('A JSON list containing classification values for a Python package.')
+    )
+    download_url = serializers.CharField(
         required=False, allow_blank=True,
-        help_text=_('The author\'s e-mail address. ')
+        help_text=_('Legacy field denoting the URL from which this package can be downloaded.')
     )
+    supported_platform = serializers.CharField(
+        required=False, allow_blank=True,
+        help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ')
+    )
+    # Version 1.2
     maintainer = serializers.CharField(
         required=False, allow_blank=True,
         help_text=_('The maintainer\'s name at a minimum; '
@@ -144,14 +144,11 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
         required=False, allow_blank=True,
         help_text=_('The maintainer\'s e-mail address.')
     )
-    license = serializers.CharField(
-        required=False, allow_blank=True,
-        help_text=_('Text indicating the license covering the distribution')
-    )
-    requires_python = serializers.CharField(
-        required=False, allow_blank=True,
-        help_text=_('The Python version(s) that the distribution is guaranteed to be '
-                    'compatible with.')
+    obsoletes_dist = serializers.JSONField(
+        required=False, default=list,
+        help_text=_('A JSON list containing names of a distutils project\'s distribution which '
+                    'this distribution renders obsolete, meaning that the two projects should not '
+                    'be installed at the same time.')
     )
     project_url = serializers.CharField(
         required=False, allow_blank=True,
@@ -161,39 +158,73 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
         required=False, default=dict,
         help_text=_('A dictionary of labels and URLs for the project.')
     )
-    platform = serializers.CharField(
-        required=False, allow_blank=True,
-        help_text=_('A comma-separated list of platform specifications, '
-                    'summarizing the operating systems supported by the package.')
+    provides_dist = serializers.JSONField(
+        required=False, default=list,
+        help_text=_('A JSON list containing names of a Distutils project which is contained'
+                    ' within this distribution.')
     )
-    supported_platform = serializers.CharField(
-        required=False, allow_blank=True,
-        help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ')
+    requires_external = serializers.JSONField(
+        required=False, default=list,
+        help_text=_('A JSON list containing some dependency in the system that the distribution '
+                    'is to be used.')
     )
     requires_dist = serializers.JSONField(
         required=False, default=list,
         help_text=_('A JSON list containing names of some other distutils project '
                     'required by this distribution.')
     )
-    provides_dist = serializers.JSONField(
-        required=False, default=list,
-        help_text=_('A JSON list containing names of a Distutils project which is contained'
-                    ' within this distribution.')
+    requires_python = serializers.CharField(
+        required=False, allow_blank=True,
+        help_text=_('The Python version(s) that the distribution is guaranteed to be '
+                    'compatible with.')
     )
-    obsoletes_dist = serializers.JSONField(
+    # Version 2.1
+    description_content_type = serializers.CharField(
+        required=False, allow_blank=True,
+        help_text=_('A string stating the markup syntax (if any) used in the distribution’s'
+                    ' description, so that tools can intelligently render the description.')
+    )
+    provides_extras = serializers.JSONField(
         required=False, default=list,
-        help_text=_('A JSON list containing names of a distutils project\'s distribution which '
-                    'this distribution renders obsolete, meaning that the two projects should not '
-                    'be installed at the same time.')
+        help_text=_('A JSON list containing names of optional features provided by the package.')
     )
-    requires_external = serializers.JSONField(
+    # Version 2.2
+    dynamic = serializers.JSONField(
         required=False, default=list,
-        help_text=_('A JSON list containing some dependency in the system that the distribution '
-                    'is to be used.')
+        help_text=_('A JSON list containing names of other core metadata fields which are '
+                    'permitted to vary between sdist and bdist packages. Fields NOT marked '
+                    'dynamic MUST be the same between bdist and sdist.')
     )
-    classifiers = serializers.JSONField(
+    # Version 2.4
+    license_expression = serializers.CharField(
+        required=False, allow_blank=True,
+        help_text=_('Text string that is a valid SPDX license expression.')
+    )
+    license_file = serializers.JSONField(
         required=False, default=list,
-        help_text=_('A JSON list containing classification values for a Python package.')
+        help_text=_('A JSON list containing names of the paths to license-related files.')
+    )
+    # Release metadata
+    filename = serializers.CharField(
+        help_text=_('The name of the distribution package, usually of the format:'
+                    ' {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}'
+                    '-{platform tag}.{packagetype}'),
+        read_only=True,
+    )
+    packagetype = serializers.CharField(
+        help_text=_('The type of the distribution package '
+                    '(e.g. sdist, bdist_wheel, bdist_egg, etc)'),
+        read_only=True,
+    )
+    python_version = serializers.CharField(
+        help_text=_(
+            'The tag that indicates which Python implementation or version the package requires.'
+        ),
+        read_only=True,
+    )
+    sha256 = serializers.CharField(
+        default='',
+        help_text=_('The SHA256 digest of this package.'),
     )
 
     def deferred_validate(self, data):
@@ -242,11 +273,13 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
 
     class Meta:
         fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + (
-            'filename', 'packagetype', 'name', 'version', 'sha256', 'metadata_version', 'summary',
-            'description', 'description_content_type', 'keywords', 'home_page', 'download_url',
-            'author', 'author_email', 'maintainer', 'maintainer_email', 'license',
-            'requires_python', 'project_url', 'project_urls', 'platform', 'supported_platform',
-            'requires_dist', 'provides_dist', 'obsoletes_dist', 'requires_external', 'classifiers'
+            'author', 'author_email', 'description', 'home_page', 'keywords', 'license',
+            'metadata_version', 'name', 'platform', 'summary', 'version', 'classifiers',
+            'download_url', 'supported_platform', 'maintainer', 'maintainer_email',
+            'obsoletes_dist', 'project_url', 'project_urls', 'provides_dist', 'requires_external',
+            'requires_dist', 'requires_python', 'description_content_type',
+            'provides_extras', 'dynamic', 'license_expression', 'license_file',
+            'filename', 'packagetype', 'python_version', 'sha256'
         )
         model = python_models.PythonPackageContent
 

pulp_python-3.17.0/pulp_python/app/tasks/repair.py

@@ -0,0 +1,198 @@
+import logging
+from collections import defaultdict
+from gettext import gettext as _
+from itertools import groupby
+from uuid import UUID
+
+from django.db.models import Prefetch
+from django.db.models.query import QuerySet
+from pulp_python.app.models import PythonPackageContent, PythonRepository
+from pulp_python.app.utils import (
+    artifact_to_python_content_data,
+    fetch_json_release_metadata,
+    parse_metadata,
+)
+from pulpcore.plugin.models import ContentArtifact, ProgressReport
+from pulpcore.plugin.util import get_domain
+
+log = logging.getLogger(__name__)
+
+
+BULK_SIZE = 1000
+
+
+def repair(repository_pk: UUID) -> None:
+    """
+    Repairs metadata of all packages for the specified repository.
+
+    Args:
+        repository_pk (UUID): The primary key of the repository to repair.
+
+    Returns:
+        None
+    """
+    repository = PythonRepository.objects.get(pk=repository_pk)
+
+    log.info(
+        _(
+            "Repairing packages' metadata for the latest version of repository {}."
+        ).format(repository.name)
+    )
+    content_set = repository.latest_version().content.values_list("pk", flat=True)
+    content = PythonPackageContent.objects.filter(pk__in=content_set)
+
+    num_repaired, pkgs_not_repaired = repair_metadata(content)
+    log.info(
+        _(
+            "{} packages' metadata repaired. Not repaired packages due to either "
+            "inaccessible URL or mismatched sha256: {}."
+        ).format(num_repaired, pkgs_not_repaired)
+    )
+
+
+def repair_metadata(content: QuerySet[PythonPackageContent]) -> tuple[int, set[str]]:
+    """
+    Repairs metadata for a queryset of PythonPackageContent objects
+    and updates the progress report.
+
+    Args:
+        content (QuerySet[PythonPackageContent]): The queryset of items to repair.
+
+    Returns:
+        tuple[int, set[str]]: A tuple containing:
+            - The number of packages that were repaired.
+            - A set of packages' PKs that were not repaired.
+    """
+    immediate_content = (
+        content.filter(contentartifact__artifact__isnull=False)
+        .distinct()
+        .prefetch_related("_artifacts")
+    )
+    on_demand_content = (
+        content.filter(contentartifact__artifact__isnull=True)
+        .distinct()
+        .prefetch_related(
+            Prefetch(
+                "contentartifact_set",
+                queryset=ContentArtifact.objects.prefetch_related("remoteartifact_set"),
+            )
+        )
+        .order_by("name", "version")
+    )
+    domain = get_domain()
+
+    batch = []
+    set_of_update_fields = set()
+    total_repaired = 0
+    # Keep track of on-demand packages that were not repaired
+    pkgs_not_repaired = set()
+
+    progress_report = ProgressReport(
+        message="Repairing packages' metadata",
+        code="repair.metadata",
+        total=content.count(),
+    )
+    progress_report.save()
+    with progress_report:
+        for package in progress_report.iter(
+            immediate_content.iterator(chunk_size=BULK_SIZE)
+        ):
+            new_data = artifact_to_python_content_data(
+                package.filename, package._artifacts.get(), domain
+            )
+            total_repaired += update_package_if_needed(
+                package, new_data, batch, set_of_update_fields
+            )
+
+        # For on-demand content, we expect that:
+        # 1. PythonPackageContent always has correct name and version
+        # 2. RemoteArtifact always has correct sha256
+        for (name, version), group in groupby(
+            on_demand_content.iterator(chunk_size=BULK_SIZE),
+            key=lambda x: (x.name, x.version),
+        ):
+            group_set = set(group)
+            grouped_by_url = defaultdict(list)
+
+            for package in group_set:
+                for ra in package.contentartifact_set.get().remoteartifact_set.all():
+                    grouped_by_url[ra.remote.url].append((package, ra))
+
+            # Prioritize the URL that can serve the most packages
+            for url, pkg_ra_pairs in sorted(
+                grouped_by_url.items(), key=lambda x: len(x[1]), reverse=True
+            ):
+                if not group_set:
+                    break  # No packages left to repair, move onto the next group
+                remotes = set([pkg_ra[1].remote for pkg_ra in pkg_ra_pairs])
+                try:
+                    json_data = fetch_json_release_metadata(name, version, remotes)
+                except Exception:
+                    continue
+
+                for package, ra in pkg_ra_pairs:
+                    if package not in group_set:
+                        continue  # Package was already repaired
+                    # Extract data only for the specific distribution being checked
+                    dist_data = None
+                    for dist in json_data["urls"]:
+                        if ra.sha256 == dist["digests"]["sha256"]:
+                            dist_data = dist
+                            break
+                    if not dist_data:
+                        continue
+
+                    new_data = parse_metadata(json_data["info"], version, dist_data)
+                    new_data.pop("url")  # url belongs to RemoteArtifact
+                    total_repaired += update_package_if_needed(
+                        package, new_data, batch, set_of_update_fields
+                    )
+                    group_set.remove(package)
+                    progress_report.increment()
+            # Store and track the unrepaired packages after all URLs are processed
+            pkgs_not_repaired.update([p.pk for p in group_set])
+            progress_report.increase_by(len(group_set))
+
+    if batch:
+        total_repaired += len(batch)
+        PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
+
+    return total_repaired, pkgs_not_repaired
+
+
+def update_package_if_needed(
+    package: PythonPackageContent,
+    new_data: dict,
+    batch: list[PythonPackageContent],
+    set_of_update_fields: set[str],
+) -> int:
+    """
+    Compares the current package data with new data and updates the package
+    if needed ("batch" and "set_of_update_fields" are updated in-place).
+
+    Args:
+        package: Package to check and update.
+        new_data: A dict of new field values to compare against the package.
+        batch: A list of packages that were updated.
+        set_of_update_fields: A set of package field names that were updated.
+
+    Returns:
+        The count of repaired packages (increments in multiples of BULK_SIZE only).
+    """
+    total_repaired = 0
+    changed = False
+    for field, value in new_data.items():
+        if getattr(package, field) != value:
+            setattr(package, field, value)
+            set_of_update_fields.add(field)
+            changed = True
+    if changed:
+        batch.append(package)
+
+    if len(batch) == BULK_SIZE:
+        PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
+        total_repaired += BULK_SIZE
+        batch.clear()
+        set_of_update_fields.clear()
+
+    return total_repaired