pulp-ansible 0.25.0__tar.gz → 0.25.2__tar.gz

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/CHANGES.md

@@ -8,6 +8,26 @@
 
 [//]: # (towncrier release notes start)
 
+## 0.25.2 (2026-01-27) {: #0.25.2 }
+
+#### Bugfixes {: #0.25.2-bugfix }
+
+- Exclude some bloated fields by default from the collection version views.
+- Fixed a bug that made synching git repositories fail on clone.
+- Fixed an incompatibility with Django5.
+- Fixed the sync pipeline to not manipulate the RepositoryContent table directly when undeprecating collections.
+
+---
+
+## 0.25.1 (2025-05-06) {: #0.25.1 }
+
+#### Bugfixes {: #0.25.1-bugfix }
+
+- Fixed broken collection import logs.
+- Made collection sync slightly more resiliant in face of missing "next" links.
+
+---
+
 ## 0.25.0 (2025-04-23) {: #0.25.0 }
 
 #### Features {: #0.25.0-feature }

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulp-ansible
-Version: 0.25.0
+Version: 0.25.2
 Summary: Pulp plugin to manage Ansible content, e.g. roles
 Author-email: Pulp Ansible Plugin Project Developers <pulp-dev@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -18,13 +18,12 @@ Classifier: Programming Language :: Python :: 3.9
 Requires-Python: >=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: async_lru<2.1,>=1.0
-Requires-Dist: galaxy_importer<0.5,>=0.4.5
+Requires-Dist: galaxy_importer<0.5,>=0.4.27
 Requires-Dist: GitPython<3.2,>=3.1.24
-Requires-Dist: jsonschema<4.24,>=4.9
-Requires-Dist: Pillow<11.2,>=10.3
+Requires-Dist: jsonschema<4.26,>=4.9
+Requires-Dist: Pillow<11.4,>=10.3
 Requires-Dist: pulpcore<3.85,>=3.49.0
-Requires-Dist: PyYAML<7.0,>=5.4.1
+Requires-Dist: PyYAML<7.0,>=6.0.1
 Requires-Dist: semantic_version<2.11,>=2.9
 Dynamic: license-file
 

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/pulp_ansible/app/__init__.py

@@ -8,5 +8,5 @@ class PulpAnsiblePluginAppConfig(PulpPluginAppConfig):
 
     name = "pulp_ansible.app"
     label = "ansible"
-    version = "0.25.0"
+    version = "0.25.2"
     python_package_name = "pulp-ansible"

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/pulp_ansible/app/migrations/0057_collectionversion_sha256_migrate.py

@@ -16,11 +16,11 @@ def add_sha256_to_current_models(apps, schema_editor):
 
     for collection_version in (
         CollectionVersion.objects.prefetch_related(
-            "content_artifacts", "content_artifacts__artifact"
+            "contentartifact_set", "contentartifact_set__artifact"
         )
         .filter(sha256="")
         .only("pk", "sha256")
-        .iterator()
+        .iterator(chunk_size=2000)
     ):
         content_artifact = collection_version.contentartifact_set.get()
         if content_artifact.artifact:

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/pulp_ansible/app/serializers.py

@@ -658,6 +658,22 @@ class CollectionVersionSerializer(ContentChecksumSerializer, CollectionVersionUp
 
     creating = True
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        try:
+            request = self.context["request"]
+            if request.method != "GET":
+                return
+        except (AttributeError, TypeError, KeyError):
+            # The serializer was not initialized with request context.
+            return
+        query_params = request.query_params
+        if self.include_arg_name in query_params or self.exclude_arg_name in query_params:
+            return
+        for fieldname in ["files", "manifest", "docs_blob", "contents"]:
+            # Redact these fields by default as they have no size limit.
+            self.fields.pop(fieldname)
+
     def validate(self, data):
         """Run super() validate if creating, else return data."""
         # This validation is for creating CollectionVersions

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/pulp_ansible/app/settings.py

@@ -2,7 +2,7 @@ import socket
 
 LOGGING = {
     "loggers": {
-        "pulp_ansible.app.tasks.collection.import_collection": {
+        "pulp_ansible.app.tasks.upload.process_collection_artifact": {
             "level": "INFO",
             "handlers": ["collection_import"],
             "propagate": False,

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/pulp_ansible/app/tasks/collections.py

@@ -1,4 +1,5 @@
 import asyncio
+import functools
 import hashlib
 import json
 import logging
@@ -8,11 +9,11 @@ from gettext import gettext as _
 from operator import attrgetter
 from urllib.parse import urljoin
 from uuid import uuid4
+import tempfile
 
 import yaml
 from aiohttp.client_exceptions import ClientError, ClientResponseError
 from asgiref.sync import sync_to_async
-from async_lru import alru_cache
 from django.conf import settings
 from django.db import transaction
 from django.db.models import Q
@@ -32,7 +33,6 @@ from pulpcore.plugin.models import (
     ProgressReport,
     PulpTemporaryFile,
     Remote,
-    RepositoryContent,
     RepositoryVersion,
     Task,
 )
@@ -40,6 +40,8 @@ from pulpcore.plugin.stages import (
     ArtifactDownloader,
     ArtifactSaver,
     ContentSaver,
+    ContentAssociation,
+    EndStage,
     DeclarativeArtifact,
     DeclarativeContent,
     DeclarativeVersion,
@@ -49,6 +51,7 @@ from pulpcore.plugin.stages import (
     RemoteArtifactSaver,
     ResolveContentFutures,
     Stage,
+    create_pipeline,
 )
 from rest_framework.serializers import ValidationError
 from semantic_version import SimpleSpec, Version
@@ -79,6 +82,26 @@ from pulp_ansible.app.tasks.utils import (
 log = logging.getLogger(__name__)
 
 
+def async_cache(f):
+    # A simple implementation for caching an async function with hashable arguments.
+    _cache = {}
+
+    @functools.wraps(f)
+    async def _wrapped_f(*args):
+        try:
+            return _cache[args]
+        except KeyError:
+            result = await f(*args)
+            _cache[args] = result
+            if len(_cache) > 128:
+                # This is a bit ad hoc and will give fifo instead of lru behaviour.
+                # I doubt it even makes a difference here.
+                _cache.pop(next(iter(_cache.keys())))
+            return result
+
+    return _wrapped_f
+
+
 # semantic_version.SimpleSpec interpretes "*" as ">=0.0.0"
 class AnsibleSpec(SimpleSpec):
     def __init__(self, expression):
@@ -106,13 +129,15 @@ async def declarative_content_from_git_repo(remote, url, git_ref=None, metadata_
     if git_ref:
         try:
             gitrepo = Repo.clone_from(
-                url, uuid4(), depth=1, branch=git_ref, multi_options=["--recurse-submodules"]
+                url, str(uuid4()), depth=1, branch=git_ref, multi_options=["--recurse-submodules"]
             )
         except GitCommandError:
-            gitrepo = Repo.clone_from(url, uuid4(), multi_options=["--recurse-submodules"])
+            gitrepo = Repo.clone_from(url, str(uuid4()), multi_options=["--recurse-submodules"])
             gitrepo.git.checkout(git_ref)
     else:
-        gitrepo = Repo.clone_from(url, uuid4(), depth=1, multi_options=["--recurse-submodules"])
+        gitrepo = Repo.clone_from(
+            url, str(uuid4()), depth=1, multi_options=["--recurse-submodules"]
+        )
     commit_sha = gitrepo.head.commit.hexsha
     metadata, artifact_path = sync_collection(gitrepo.working_dir, ".")
     artifact = Artifact.init_and_validate(artifact_path)
@@ -196,14 +221,7 @@ def sync(remote_pk, repository_pk, mirror, optimize):
     if not remote.url:
         raise ValueError(_("A CollectionRemote must have a 'url' specified to synchronize."))
 
-    deprecation_before_sync = set()
-    for namespace, name in AnsibleCollectionDeprecated.objects.filter(
-        pk__in=repository.latest_version().content
-    ).values_list("namespace", "name"):
-        deprecation_before_sync.add(f"{namespace}.{name}")
-    first_stage = CollectionSyncFirstStage(
-        remote, repository, is_repo_remote, deprecation_before_sync, optimize
-    )
+    first_stage = CollectionSyncFirstStage(remote, repository, is_repo_remote, optimize)
     if first_stage.should_sync is False:
         log.debug(_("no-op: remote wasn't updated since last sync."))
         return
@@ -217,23 +235,6 @@ def sync(remote_pk, repository_pk, mirror, optimize):
     repository.last_synced_metadata_time = first_stage.last_synced_metadata_time
     repository.save(update_fields=["last_synced_metadata_time"])
 
-    # This goes against all rules!
-    # Content must be added and removed via the dedicated functions, which is impossible after the
-    # repository version was finalized.
-    # TODO Fix this!
-    to_undeprecate = Q()
-    undeprecated = deprecation_before_sync.difference(first_stage.deprecation_after_sync)
-    if undeprecated:
-        for collection in undeprecated:
-            namespace, name = collection.split(".")
-            to_undeprecate |= Q(namespace=namespace, name=name)
-        deprecated = AnsibleCollectionDeprecated.objects.filter(to_undeprecate)
-        RepositoryContent.objects.filter(
-            repository=repository,
-            content__in=deprecated,
-            version_removed__isnull=True,
-        ).update(version_removed=repo_version)
-
 
 def import_collection(
     temp_file_pk,
@@ -478,13 +479,58 @@ class AnsibleDeclarativeVersion(DeclarativeVersion):
 
         return pipeline
 
+    def create(self):
+        """
+        Perform the work. This is the long-blocking call where all syncing occurs.
+
+        Returns: The created RepositoryVersion or None if it represents no change from the latest.
+        """
+        with tempfile.TemporaryDirectory(dir="."):
+            with self.repository.new_version() as new_version:
+                deprecation_before_sync = {
+                    (namespace, name)
+                    for namespace, name in AnsibleCollectionDeprecated.objects.filter(
+                        pk__in=self.repository.latest_version().content
+                    ).values_list("namespace", "name")
+                }
+                loop = asyncio.get_event_loop()
+                stages = self.pipeline_stages(new_version)
+                stages.append(UndeprecateStage(deprecation_before_sync))
+                stages.append(ContentAssociation(new_version, self.mirror))
+                stages.append(EndStage())
+                pipeline = create_pipeline(stages)
+                loop.run_until_complete(pipeline)
+
+                if deprecation_before_sync:
+                    to_undeprecate = Q()
+                    for namespace, name in deprecation_before_sync:
+                        to_undeprecate |= Q(namespace=namespace, name=name)
+                    new_version.remove_content(
+                        AnsibleCollectionDeprecated.objects.filter(to_undeprecate)
+                    )
+
+        return new_version if new_version.complete else None
+
+
+class UndeprecateStage(Stage):
+    def __init__(self, deprecation_before_sync):
+        self.deprecation_before_sync = deprecation_before_sync
+
+    async def run(self):
+        async for batch in self.batches():
+            for d_content in batch:
+                if isinstance(d_content.content, AnsibleCollectionDeprecated):
+                    key = (d_content.content.namespace, d_content.content.name)
+                    self.deprecation_before_sync.discard(key)
+                await self.put(d_content)
+
 
 class CollectionSyncFirstStage(Stage):
     """
     The first stage of a pulp_ansible sync pipeline.
     """
 
-    def __init__(self, remote, repository, is_repo_remote, deprecation_before_sync, optimize):
+    def __init__(self, remote, repository, is_repo_remote, optimize):
         """
         The first stage of a pulp_ansible sync pipeline.
 
@@ -492,15 +538,12 @@ class CollectionSyncFirstStage(Stage):
             remote (CollectionRemote): The remote data to be used when syncing
             repository (AnsibleRepository): The repository being syncedself.
             is_repo_remote (bool): True if the remote is the repository's remote.
-            deprecation_before_sync (set): Set of deprecations before the sync.
             optimize (boolean): Whether to optimize sync or not.
 
         """
         super().__init__()
         self.remote = remote
         self.repository = repository
-        self.deprecation_before_sync = deprecation_before_sync
-        self.deprecation_after_sync = set()
         self.collection_info = parse_collections_requirements_file(remote.requirements_file)
         self.exclude_info = {}
         self.add_dependents = self.collection_info and self.remote.sync_dependencies
@@ -521,7 +564,7 @@ class CollectionSyncFirstStage(Stage):
             self._should_we_sync()
         )
 
-    @alru_cache(maxsize=128)
+    @async_cache
     async def _get_root_api(self, root):
         """
         Returns the root api path and api version.
@@ -557,7 +600,7 @@ class CollectionSyncFirstStage(Stage):
 
         return endpoint, api_version
 
-    @alru_cache(maxsize=128)
+    @async_cache
     async def _get_paginated_collection_api(self, root):
         """
         Returns the collection api path and api version.
@@ -767,7 +810,6 @@ class CollectionSyncFirstStage(Stage):
                         d_content = DeclarativeContent(
                             content=AnsibleCollectionDeprecated(namespace=namespace, name=name),
                         )
-                        self.deprecation_after_sync.add(f"{namespace}.{name}")
                         await self.put(d_content)
                     tasks.append(
                         loop.create_task(
@@ -802,7 +844,6 @@ class CollectionSyncFirstStage(Stage):
             d_content = DeclarativeContent(
                 content=AnsibleCollectionDeprecated(namespace=namespace, name=name),
             )
-            self.deprecation_after_sync.add(f"{namespace}.{name}")
             await self.put(d_content)
 
         all_versions_of_collection = self._unpaginated_collection_version_metadata[namespace][name]
@@ -848,9 +889,11 @@ class CollectionSyncFirstStage(Stage):
     @staticmethod
     def _get_response_next_value(api_version, response):
         if api_version == 2:
-            return response["next"]
+            return response.get("next")
+        elif links := response.get("links"):
+            return links.get("next")
         else:
-            return response["links"]["next"]
+            return None
 
     def _collection_list_downloader(self, api_version, collection_endpoint, page_num, page_size):
         if api_version == 2:
@@ -923,9 +966,6 @@ class CollectionSyncFirstStage(Stage):
                             namespace=collection["namespace"], name=collection["name"]
                         ),
                     )
-                    self.deprecation_after_sync.add(
-                        f"{collection['namespace']}.{collection['name']}"
-                    )
                     await self.put(d_content)
 
         for collections_in_namespace in self._unpaginated_collection_version_metadata.values():

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/pulp_ansible/tests/functional/api/collection/v3/test_deprecation.py

@@ -35,7 +35,6 @@ def test_deprecation(
     collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
         first_distribution.base_path, namespace="testing"
     )
-
     assert collections.data[0].deprecated
     collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
         first_distribution.base_path, namespace="pulp"
@@ -62,15 +61,12 @@ def test_deprecation(
     )
     assert collections.data[0].deprecated
 
-    # Change the deprecated status for the testing collection on the original repo to False
-    result = ansible_bindings.PulpAnsibleApiV3CollectionsApi.update(
-        "k8s_demo_collection", "testing", first_distribution.base_path, {"deprecated": False}
-    )
-    monitor_task(result.task)
+    # Sync again to see if the deprecated state is kept.
+    ansible_sync_factory(second_repo, optimize=False)
     collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
-        first_distribution.base_path, namespace="testing"
+        second_distribution.base_path, namespace="testing"
     )
-    assert not collections.data[0].deprecated
+    assert collections.data[0].deprecated
 
     # Update the requirements to sync down both collections this time
     requirements = (
@@ -83,9 +79,34 @@ def test_deprecation(
     # Sync the second repo again
     second_repo = ansible_sync_factory(second_repo)
 
+    # Assert the state of deprecated True for testing, False for pulp
+    collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
+        second_distribution.base_path, namespace="testing"
+    )
+    assert collections.data[0].deprecated
+    collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
+        second_distribution.base_path, namespace="pulp"
+    )
+    assert not collections.data[0].deprecated
+
+    # Change the deprecated status for the testing collection on the original repo to False
+    monitor_task(
+        ansible_bindings.PulpAnsibleApiV3CollectionsApi.update(
+            "k8s_demo_collection", "testing", first_distribution.base_path, {"deprecated": False}
+        ).task
+    )
+    collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
+        first_distribution.base_path, namespace="testing"
+    )
+    assert not collections.data[0].deprecated
+
+    # Sync the second repo again
+    second_repo = ansible_sync_factory(second_repo)
+
     # Assert both collections show deprecated=False
     collections = ansible_bindings.PulpAnsibleApiV3CollectionsApi.list(
         second_distribution.base_path
     )
+    assert len(collections.data) == 2, collections
     for collection in collections.data:
         assert not collection.deprecated

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/pulp_ansible.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pulp-ansible
-Version: 0.25.0
+Version: 0.25.2
 Summary: Pulp plugin to manage Ansible content, e.g. roles
 Author-email: Pulp Ansible Plugin Project Developers <pulp-dev@redhat.com>
 Project-URL: Homepage, https://pulpproject.org
@@ -18,13 +18,12 @@ Classifier: Programming Language :: Python :: 3.9
 Requires-Python: >=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: async_lru<2.1,>=1.0
-Requires-Dist: galaxy_importer<0.5,>=0.4.5
+Requires-Dist: galaxy_importer<0.5,>=0.4.27
 Requires-Dist: GitPython<3.2,>=3.1.24
-Requires-Dist: jsonschema<4.24,>=4.9
-Requires-Dist: Pillow<11.2,>=10.3
+Requires-Dist: jsonschema<4.26,>=4.9
+Requires-Dist: Pillow<11.4,>=10.3
 Requires-Dist: pulpcore<3.85,>=3.49.0
-Requires-Dist: PyYAML<7.0,>=5.4.1
+Requires-Dist: PyYAML<7.0,>=6.0.1
 Requires-Dist: semantic_version<2.11,>=2.9
 Dynamic: license-file
 

pulp_ansible-0.25.2/pulp_ansible.egg-info/requires.txt

@@ -0,0 +1,7 @@
+galaxy_importer<0.5,>=0.4.27
+GitPython<3.2,>=3.1.24
+jsonschema<4.26,>=4.9
+Pillow<11.4,>=10.3
+pulpcore<3.85,>=3.49.0
+PyYAML<7.0,>=6.0.1
+semantic_version<2.11,>=2.9

{pulp_ansible-0.25.0 → pulp_ansible-0.25.2}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = "pulp-ansible"
-version = "0.25.0"
+version = "0.25.2"
 description = "Pulp plugin to manage Ansible content, e.g. roles"
 readme = "README.rst"
 authors = [
@@ -24,13 +24,12 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
-  "async_lru>=1.0,<2.1",
-  "galaxy_importer>=0.4.5,<0.5",
+  "galaxy_importer>=0.4.27,<0.5",
   "GitPython>=3.1.24,<3.2",
-  "jsonschema>=4.9,<4.24",
-  "Pillow>=10.3,<11.2",
+  "jsonschema>=4.9,<4.26",
+  "Pillow>=10.3,<11.4",
   "pulpcore>=3.49.0,<3.85",
-  "PyYAML>=5.4.1,<7.0",
+  "PyYAML>=6.0.1,<7.0",
   "semantic_version>=2.9,<2.11",
 ]
 
@@ -101,7 +100,7 @@ ignore = [
 [tool.bumpversion]
 # This section is managed by the plugin template. Do not edit manually.
 
-current_version = "0.25.0"
+current_version = "0.25.2"
 commit = false
 tag = false
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<alpha>0a)?(?P<patch>\\d+)(\\.(?P<release>[a-z]+))?"

pulp_ansible-0.25.0/pulp_ansible.egg-info/requires.txt

@@ -1,8 +0,0 @@
-async_lru<2.1,>=1.0
-galaxy_importer<0.5,>=0.4.5
-GitPython<3.2,>=3.1.24
-jsonschema<4.24,>=4.9
-Pillow<11.2,>=10.3
-pulpcore<3.85,>=3.49.0
-PyYAML<7.0,>=5.4.1
-semantic_version<2.11,>=2.9